24 lines
647 B
Python
24 lines
647 B
Python
from bs4 import BeautifulSoup
|
|
from driver import Driven
|
|
|
|
class ChomperBase(Driven):
|
|
'''
|
|
Web parser/ingester base class
|
|
|
|
Attributes:
|
|
soup (TYPE): #DOC#
|
|
'''
|
|
def __init__(self, new_driver=False):
|
|
localArgs = {k:v for k,v in locals().items() if k is not 'self'}
|
|
super(ChomperBase,self).__init__(**localArgs)
|
|
|
|
def main(self, url):
|
|
self.get(url)
|
|
self.make_soup()
|
|
return self.parse()
|
|
|
|
def make_soup(self):
|
|
self.soup = BeautifulSoup(self.driver.page_source,'lxml')
|
|
|
|
def parse(self):
|
|
raise NotImplementedError('Inheriting class must implement parse_soup!') |