Geen omschrijving

parse.py 3.8KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495
  1. import pandas as pd
  2. import json
  3. from os import remove
  4. def initial_parse(name, url):
  5. table_MN = pd.read_html(url)
  6. df = table_MN[0]
  7. df.columns = ['Curso', 'Nombre', 'c', 'Dias', 'Hora', 'Salon']
  8. upi = df[['Curso', 'Nombre', 'Dias', 'Hora', 'Salon']]
  9. # print(upi.head())
  10. # print(upi.to_json(orient='records'))
  11. # puedes usar esta alternativa o la de la linea 17
  12. # upi.to_json(r'C:\Users\diego\Documents\miupi_parse\soup\file.json')
  13. upi.to_json(f'{name}_temp.json', orient='records')
  14. def final_parse(name):
  15. with open(f"{name}_temp.json") as f_in:
  16. data = json.load(f_in)
  17. newdata = []
  18. for item in data:
  19. if(item["Curso"] == None):
  20. pass
  21. else:
  22. course = {}
  23. course["Curso"] = item["Curso"][0:8]
  24. course["Seccion"] = item["Curso"][9:12]
  25. info = item["Nombre"].split("Profesor: ")
  26. if((profesor := info[1][0:-11]) != ""):
  27. course["Profesor"] = profesor
  28. else:
  29. course["Profesor"] = "TBA"
  30. course["Nombre"] = info[0][0:-1]
  31. course["Dias"] = item["Dias"].split(" ")
  32. horario = []
  33. if(item["Hora"] != None):
  34. for i in range(len(course["Dias"])):
  35. horario.append(item["Hora"][15*i:15*(i+1)])
  36. else:
  37. horario.append("TBA")
  38. course["Horario"] = horario
  39. salones_raw = item["Salon"].split("Edificio: ")
  40. salones = []
  41. for i in salones_raw:
  42. if(i != "-" and i != "- " and i != ""):
  43. salones.append(i[:-1])
  44. course["Salones"] = salones
  45. course["Info_Extra"] = item["Curso"][13:-1]
  46. newdata.append(course)
  47. with open(f'{name}.json', 'w') as f_out:
  48. json.dump(newdata, f_out)
  49. if __name__ == "__main__":
  50. faculty_urls = {"Asuntos_Aca": "https://miupi.uprrp.edu/horarios/RBA120_AA.HTML",
  51. "Admi": "https://miupi.uprrp.edu/horarios/RBA120_AE.HTML",
  52. "Admi_Grad": "https://miupi.uprrp.edu/horarios/RBA120_AE2.HTML",
  53. "Arqui": "https://miupi.uprrp.edu/horarios/RBA120_AQ.HTML",
  54. "Arqui_Grad": "https://miupi.uprrp.edu/horarios/RBA120_AQ2.HTML",
  55. "Ciencia_Tech_Info_Grad": "https://miupi.uprrp.edu/horarios/RBA120_CB.HTML",
  56. "Ciencias_Militares": "https://miupi.uprrp.edu/horarios/RBA120_CM.HTML",
  57. "Naturales": "https://miupi.uprrp.edu/horarios/RBA120_CN.HTML",
  58. "Naturales_Grad": "https://miupi.uprrp.edu/horarios/RBA120_CN2.HTML",
  59. "Comunicaciones": "https://miupi.uprrp.edu/horarios/RBA120_CP.HTML",
  60. "Comunicaciones_Grad": "https://miupi.uprrp.edu/horarios/RBA120_CP2.HTML",
  61. "Sociales": "https://miupi.uprrp.edu/horarios/RBA120_CS.HTML",
  62. "Sociales_Grad": "https://miupi.uprrp.edu/horarios/RBA120_CS2.HTML ",
  63. "Derecho": "https://miupi.uprrp.edu/horarios/RBA120_DE.HTML",
  64. "Educacion_Continua": "https://miupi.uprrp.edu/horarios/RBA120_EC.HTML",
  65. "Educacion": "https://miupi.uprrp.edu/horarios/RBA120_ED.HTML",
  66. "Educacion_Grad": "https://miupi.uprrp.edu/horarios/RBA120_ED2.HTML",
  67. "Generales": "https://miupi.uprrp.edu/horarios/RBA120_EG.HTML",
  68. "Humanidades": "https://miupi.uprrp.edu/horarios/RBA120_HU.HTML",
  69. "Humanidades_Grad": "https://miupi.uprrp.edu/horarios/RBA120_HU2.HTML",
  70. "Planificacion": "https://miupi.uprrp.edu/horarios/RBA120_PL.HTML"}
  71. #initial_parse("Naturales", faculty_urls["Naturales"])
  72. for key in faculty_urls:
  73. initial_parse(key, faculty_urls[key])
  74. final_parse(key)
  75. remove(f"{key}_temp.json")