TheTechRobo
/
placescraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Allow for warcprox usage in main script

This commit is contained in:
TheTechRobo 2022-04-03 18:40:56 -04:00
parent 2bd9f18629
commit 4a98f86847
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
script.py
View File

@ -1,4 +1,4 @@
import asyncio, websockets,time,json, requests, rethinkdb, aiohttp, sys
import asyncio, websockets,time,json, requests, rethinkdb, aiohttp, sys, os
from rethinkdb import r
import raw_data
@ -58,10 +58,13 @@ class PlaceScraper:
await self.add_to_db(self.db, "sent_ws", {"sent": send, "time": time.time()})
async def run_forever(self):
global STOP, STOPPED
connector = aiohttp.TCPConnector(limit=25)
cert = False
if type(os.environ.get("DISABLE_CERT_VERIF")) is None:
cert = True
connector = aiohttp.TCPConnector(limit=50, verify_ssl=cert)
async with websockets.connect("wss://gql-realtime-2.reddit.com/query", extra_headers=[["Origin", "https://hot-potato.reddit.com"], ["User-Agent", "Mozilla"]]) as websocket:
await self._start_connection(websocket)
async with aiohttp.ClientSession(connector=connector) as sesh:
async with aiohttp.ClientSession(connector=connector, trust_env=True) as sesh:
while True:
try:
response = await websocket.recv()