TheTechRobo
/
placescraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix Too many files open error once and for all!

This commit is contained in:
TheTechRobo 2022-04-01 22:33:39 -04:00
parent 708013fed0
commit 9bd4b60ba7
3 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
__pycache__/raw_data.cpython-39.pyc
View File

Binary file not shown.

2
raw_data.py
View File

@ -1,3 +1,3 @@
connection_init = r"""{"type":"connection_init","payload":{"Authorization":"Bearer 453755016066-KVjVvAltNXo2xrQ0noKgPPRSe0p9Jw"}}"""
connection_init = r"""{"type":"connection_init","payload":{"Authorization":"Bearer 453755016066-vaewl_K8AgaWqDWNpCC-FlfrTCY-tQ"}}"""
id1 = r"""{"id":"1","type":"start","payload":{"variables":{"input":{"channel":{"teamOwner":"AFD2022","category":"CONFIG"}}},"extensions":{},"operationName":"configuration","query":"subscription configuration($input: SubscribeInput!) {\n subscribe(input: $input) {\n id\n ... on BasicMessage {\n data {\n __typename\n ... on ConfigurationMessageData {\n colorPalette {\n colors {\n hex\n index\n __typename\n }\n __typename\n }\n canvasConfigurations {\n index\n dx\n dy\n __typename\n }\n canvasWidth\n canvasHeight\n __typename\n }\n }\n __typename\n }\n __typename\n }\n}\n"}}"""
id2 = r"""{"id":"2","type":"start","payload":{"variables":{"input":{"channel":{"teamOwner":"AFD2022","category":"CANVAS","tag":"0"}}},"extensions":{},"operationName":"replace","query":"subscription replace($input: SubscribeInput!) {\n subscribe(input: $input) {\n id\n ... on BasicMessage {\n data {\n __typename\n ... on FullFrameMessageData {\n __typename\n name\n timestamp\n }\n ... on DiffFrameMessageData {\n __typename\n name\n currentTimestamp\n previousTimestamp\n }\n }\n __typename\n }\n __typename\n }\n}\n"}}"""

26
script.py
View File

@ -11,17 +11,18 @@ class PlaceScraper:
db = "place"
imgtable = "img"
wstable = "ws"
token = "ff"
async def add_to_db(self, db, table, data):
while True:
try:
conn = await r.connect()
d = await r.db(db).table(table).insert(data).run(conn)
conn.close()
await conn.close(False)
return d
except rethinkdb.errors.ReqlDriverError as ename:
print(f"Retrying DB transaction... ({ename})")
try:
conn.close()
await conn.close()
except NameError:
pass
await asyncio.sleep(2)
@ -40,15 +41,28 @@ class PlaceScraper:
self.imgtable,
{"time": time.time(), "url": url, "data": data}
)
async def regen_token(self, sesh):
async with sesh.get("https://reddit.com/r/place") as f:
a = await f.text()
i = a.index('''"session":{"accessToken":"''')
self.token = a[i + 27 : i + 57] # thanks JAA for stopping me from using my .split() fuckery
async def _start_connection(self, ws):
#await ws.send(raw_data.connection_init % self.token)
await ws.send(raw_data.connection_init)
await ws.send(raw_data.id1)
await ws.send(raw_data.id2)
async def run_forever(self):
connector = aiohttp.TCPConnector(limit=25)
async with websockets.connect("wss://gql-realtime-2.reddit.com/query", extra_headers=[["Origin", "https://hot-potato.reddit.com"]]) as websocket:
await websocket.send(raw_data.connection_init)
await websocket.send(raw_data.id1)
await websocket.send(raw_data.id2)
await self._start_connection(websocket)
async with aiohttp.ClientSession(connector=connector) as sesh:
while True:
response = await websocket.recv()
try:
response = await websocket.recv()
except websockets.exceptions.ConnectionClosedOK:
await self.regen_token(sesh)
#print(raw_data.connection_init % self.token)
await self.run_forever()
print(response)
await self.add_to_db(self.db, self.wstable, {"time":time.time(), "data": response})
try: