You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

139 lines
3.5 KiB

  1. import kiss
  2. import urllib.request
  3. import signal
  4. import gzip
  5. import hashlib
  6. import requests
  7. from bs4 import BeautifulSoup
  8. call = "VA1QLE".encode()
  9. ESC = b'\x05'
  10. ACK = b'\x06'
  11. PAGE_REQUEST = b'\x07'
  12. PAGE_RESPONSE = b'\x08'
  13. PAGE_RESPONSE_END = b'\x09'
  14. CHECKSUM = b'\x10' #Splits md5 checksum from HTML
  15. RESEND_PACKET = b'\x11'
  16. Max_Packet_Length = 750
  17. Max_Connect_Retries = 5
  18. Max_Retries = 10 #Retries after a connection is established
  19. k = kiss.TCPKISS(host='localhost', port=8001)
  20. k.start()
  21. def escapeData(data): #Not very pretty
  22. return data.replace(
  23. ACK,
  24. ESC + ACK
  25. ).replace(
  26. PAGE_REQUEST,
  27. ESC + PAGE_REQUEST
  28. ).replace(
  29. PAGE_RESPONSE,
  30. ESC + PAGE_RESPONSE
  31. ).replace(
  32. PAGE_RESPONSE_END,
  33. ESC + PAGE_RESPONSE_END
  34. ).replace(
  35. CHECKSUM,
  36. ESC + CHECKSUM
  37. ).replace(
  38. RESEND_PACKET,
  39. ESC + RESEND_PACKET
  40. )
  41. def unescapeData(data):
  42. out = b''
  43. i = 0
  44. while i < len(data):
  45. if data[i] == ESC and i + 1 < len(data):
  46. if data[i + 1] in (ACK, PAGE_REQUEST, PAGE_RESPONSE, PAGE_RESPONSE_END, CHECKSUM, RESEND_PACKET):
  47. out += str(bytearray(data[i + 1]))
  48. i += 1 # Skips over the next byte
  49. else:
  50. out += str(bytearray(data[i]))
  51. else:
  52. out += str(bytearray(data[i]))
  53. i += 1
  54. return out
  55. def signal_handler(signum, frame):
  56. raise TimeoutError
  57. signal.signal(signal.SIGALRM, signal_handler)
  58. def decode_packet(pkt, detectorByte):
  59. try:
  60. (calls, body) = b''.join(pkt)[1:].split(detectorByte, 1)
  61. (src, dest) = calls.split(b'>', 1)
  62. return (src, dest, body)
  63. except ValueError: #Exception when split fails.
  64. return (None, None, None)
  65. def downloadPage(url): #Strips out images, scripts, etc. Makes it look ugly, but much quicker.
  66. try:
  67. page = requests.get(url)
  68. code = page.status_code
  69. text = page.text
  70. soup = BeautifulSoup(text, "html5lib")
  71. #Remove script tags
  72. for item in soup.findAll('script'):
  73. item.extract()
  74. #Remove img tags
  75. for item in soup.findAll('img'):
  76. item.extract()
  77. return(code, soup.encode())
  78. except requests.ConnectionError:
  79. return (503, b"Failed to connect to website")
  80. while True:
  81. frame = b''.join(k.read(readmode=False))[1:]
  82. #print("Received packet: {}".format(frame)) #DEBUG
  83. (calls, body) = frame.split(PAGE_REQUEST, 1)
  84. (src, dest) = calls.split(b'>', 1)
  85. if dest == call: #It's for this server
  86. url = body.decode()
  87. print(url)
  88. (code, html) = downloadPage(url)
  89. html = code.to_bytes(2, 'big') + html
  90. html = escapeData(gzip.compress(html, compresslevel=9)) + PAGE_RESPONSE_END #Okay, it's not HTML. But I'm already reusing the 'body' variable and HTML is good enough
  91. attempts = 0
  92. connected = False
  93. while html != b'':
  94. success = True
  95. signal.alarm(10)
  96. try:
  97. k.write(call + b'>' + src + PAGE_RESPONSE + html[0:Max_Packet_Length] + CHECKSUM + hashlib.md5(html[0:Max_Packet_Length]).hexdigest().encode())
  98. print(html[0:Max_Packet_Length]) #DEBUGGING
  99. ack = False
  100. while not ack:
  101. pkt = k.read(readmode=False)
  102. (src_ack, dest_ack, garbage) = decode_packet(pkt, ACK)
  103. resend = False
  104. if src_ack == None:
  105. (src_ack, dest_ack, garbage) = decode_packet(pkt, RESEND_PACKET)
  106. resend = True
  107. if dest_ack == call:
  108. ack = True
  109. success = not resend
  110. except TimeoutError:
  111. #Timeout. Try again
  112. success = False
  113. if success:
  114. connected = True
  115. html = html[Max_Packet_Length:]
  116. attempts = 0
  117. else:
  118. attempts += 1
  119. if (not connected) and attempts > Max_Connect_Retries:
  120. print("Failure connecting")
  121. break
  122. elif (connected) and attempts > Max_Retries:
  123. print("Failure sending packet")
  124. break
  125. signal.alarm(0) #Reset alarm