tags.
if titles:
return [(a.get('href'), a.text) for a in a_tags if a.get('href')]
return [a.get('href') for a in a_tags if a.get('href')]
def get_urls(self, doc_or_html, titles=False, regex=False):
"""`doc_or_html`s html page or doc and returns list of urls, the regex
flag indicates we don't parse via lxml and just search the html.
"""
if doc_or_html is None:
log.critical('Must extract urls from either html, text or doc!')
return []
# If we are extracting from raw text
if regex:
doc_or_html = re.sub('<[^<]+?>', ' ', str(doc_or_html))
doc_or_html = re.findall(
'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|'
'(?:%[0-9a-fA-F][0-9a-fA-F]))+', doc_or_html)
doc_or_html = [i.strip() for i in doc_or_html]
return doc_or_html or []
# If the doc_or_html is html, parse it into a root
if isinstance(doc_or_html, str):
doc = self.parser.fromstring(doc_or_html)
else:
doc = doc_or_html
return self._get_urls(doc, titles)
def get_category_urls(self, source_url, doc):
"""Inputs source lxml root and source url, extracts domain and
finds all of the top level urls, we are assuming that these are
the category urls.
cnn.com --> [cnn.com/latest, world.cnn.com, cnn.com/asia]
"""
page_urls = self.get_urls(doc)
valid_categories = []
for p_url in page_urls:
scheme = urls.get_scheme(p_url, allow_fragments=False)
domain = urls.get_domain(p_url, allow_fragments=False)
path = urls.get_path(p_url, allow_fragments=False)
if not domain and not path:
if self.config.verbose:
print('elim category url %s for no domain and path'
% p_url)
continue
if path and path.startswith('#'):
if self.config.verbose:
print('elim category url %s path starts with #' % p_url)
continue
if scheme and (scheme != 'http' and scheme != 'https'):
if self.config.verbose:
print(('elim category url %s for bad scheme, '
'not http nor https' % p_url))
continue
if domain:
child_tld = tldextract.extract(p_url)
domain_tld = tldextract.extract(source_url)
child_subdomain_parts = child_tld.subdomain.split('.')
subdomain_contains = False
for part in child_subdomain_parts:
if part == domain_tld.domain:
if self.config.verbose:
print(('subdomain contains at %s and %s' %
(str(part), str(domain_tld.domain))))
subdomain_contains = True
break
# Ex. microsoft.com is definitely not related to
# espn.com, but espn.go.com is probably related to espn.com
if not subdomain_contains and \
(child_tld.domain != domain_tld.domain):
if self.config.verbose:
print(('elim category url %s for domain '
'mismatch' % p_url))
continue
elif child_tld.subdomain in ['m', 'i']:
if self.config.verbose:
print(('elim category url %s for mobile '
'subdomain' % p_url))
continue
else:
valid_categories.append(scheme + '://' + domain)
# TODO account for case where category is in form
# http://subdomain.domain.tld/category/ <-- still legal!
else:
# we want a path with just one subdir
# cnn.com/world and cnn.com/world/ are both valid_categories
path_chunks = [x for x in path.split('/') if len(x) > 0]
if 'index.html' in path_chunks:
path_chunks.remove('index.html')
if len(path_chunks) == 1 and len(path_chunks[0]) < 14:
valid_categories.append(domain + path)
else:
if self.config.verbose:
print(('elim category url %s for >1 path chunks '
'or size path chunks' % p_url))
stopwords = [
'about', 'help', 'privacy', 'legal', 'feedback', 'sitemap',
'profile', 'account', 'mobile', 'sitemap', 'facebook', 'myspace',
'twitter', 'linkedin', 'bebo', 'friendster', 'stumbleupon',
'youtube', 'vimeo', 'store', 'mail', 'preferences', 'maps',
'password', 'imgur', 'flickr', 'search', 'subscription', 'itunes',
'siteindex', 'events', 'stop', 'jobs', 'careers', 'newsletter',
'subscribe', 'academy', 'shopping', 'purchase', 'site-map',
'shop', 'donate', 'newsletter', 'product', 'advert', 'info',
'tickets', 'coupons', 'forum', 'board', 'archive', 'browse',
'howto', 'how to', 'faq', 'terms', 'charts', 'services',
'contact', 'plus', 'admin', 'login', 'signup', 'register',
'developer', 'proxy']
_valid_categories = []
# TODO Stop spamming urlparse and tldextract calls...
for p_url in valid_categories:
path = urls.get_path(p_url)
subdomain = tldextract.extract(p_url).subdomain
conjunction = path + ' ' + subdomain
bad = False
for badword in stopwords:
if badword.lower() in conjunction.lower():
if self.config.verbose:
print(('elim category url %s for subdomain '
'contain stopword!' % p_url))
bad = True
break
if not bad:
_valid_categories.append(p_url)
_valid_categories.append('/') # add the root
for i, p_url in enumerate(_valid_categories):
if p_url.startswith('://'):
p_url = 'http' + p_url
_valid_categories[i] = p_url
elif p_url.startswith('//'):
p_url = 'http:' + p_url
_valid_categories[i] = p_url
if p_url.endswith('/'):
p_url = p_url[:-1]
_valid_categories[i] = p_url
_valid_categories = list(set(_valid_categories))
category_urls = [urls.prepare_url(p_url, source_url)
for p_url in _valid_categories]
category_urls = [c for c in category_urls if c is not None]
return category_urls
def extract_tags(self, doc):
if len(list(doc)) == 0:
return NO_STRINGS
elements = self.parser.css_select(
doc, A_REL_TAG_SELECTOR)
if not elements:
elements = self.parser.css_select(
doc, A_HREF_TAG_SELECTOR)
if not elements:
return NO_STRINGS
tags = []
for el in elements:
tag = self.parser.getText(el)
if tag:
tags.append(tag)
return set(tags)
def calculate_best_node(self, doc):
top_node = None
nodes_to_check = self.nodes_to_check(doc)
starting_boost = float(1.0)
cnt = 0
i = 0
parent_nodes = []
nodes_with_text = []
for node in nodes_to_check:
text_node = self.parser.getText(node)
word_stats = self.stopwords_class(language=self.language). \
get_stopword_count(text_node)
high_link_density = self.is_highlink_density(node)
if word_stats.get_stopword_count() > 2 and not high_link_density:
nodes_with_text.append(node)
nodes_number = len(nodes_with_text)
negative_scoring = 0
bottom_negativescore_nodes = float(nodes_number) * 0.25
for node in nodes_with_text:
boost_score = float(0)
# boost
if self.is_boostable(node):
if cnt >= 0:
boost_score = float((1.0 / starting_boost) * 50)
starting_boost += 1
# nodes_number
if nodes_number > 15:
if (nodes_number - i) <= bottom_negativescore_nodes:
booster = float(
bottom_negativescore_nodes - (nodes_number - i))
boost_score = float(-pow(booster, float(2)))
negscore = abs(boost_score) + negative_scoring
if negscore > 40:
boost_score = float(5)
text_node = self.parser.getText(node)
word_stats = self.stopwords_class(language=self.language). \
get_stopword_count(text_node)
upscore = int(word_stats.get_stopword_count() + boost_score)
parent_node = self.parser.getParent(node)
self.update_score(parent_node, upscore)
self.update_node_count(parent_node, 1)
if parent_node not in parent_nodes:
parent_nodes.append(parent_node)
# Parent of parent node
parent_parent_node = self.parser.getParent(parent_node)
if parent_parent_node is not None:
self.update_node_count(parent_parent_node, 1)
self.update_score(parent_parent_node, upscore / 2)
if parent_parent_node not in parent_nodes:
parent_nodes.append(parent_parent_node)
cnt += 1
i += 1
top_node_score = 0
for e in parent_nodes:
score = self.get_score(e)
if score > top_node_score:
top_node = e
top_node_score = score
if top_node is None:
top_node = e
return top_node
def is_boostable(self, node):
"""A lot of times the first paragraph might be the caption under an image
so we'll want to make sure if we're going to boost a parent node that
it should be connected to other paragraphs, at least for the first n
paragraphs so we'll want to make sure that the next sibling is a
paragraph and has at least some substantial weight to it.
"""
para = "p"
steps_away = 0
minimum_stopword_count = 5
max_stepsaway_from_node = 3
nodes = self.walk_siblings(node)
for current_node in nodes:
#
current_node_tag = self.parser.getTag(current_node)
if current_node_tag == para:
if steps_away >= max_stepsaway_from_node:
return False
paragraph_text = self.parser.getText(current_node)
word_stats = self.stopwords_class(language=self.language). \
get_stopword_count(paragraph_text)
if word_stats.get_stopword_count() > minimum_stopword_count:
return True
steps_away += 1
return False
def walk_siblings(self, node):
return self.parser.previousSiblings(node)
def add_siblings(self, top_node):
baseline_score_siblings_para = self.get_siblings_score(top_node)
results = self.walk_siblings(top_node)
for current_node in results:
ps = self.get_siblings_content(
current_node, baseline_score_siblings_para)
for p in ps:
top_node.insert(0, p)
return top_node
def get_siblings_content(
self, current_sibling, baseline_score_siblings_para):
"""Adds any siblings that may have a decent score to this node
"""
if current_sibling.tag == 'p' and \
len(self.parser.getText(current_sibling)) > 0:
e0 = current_sibling
if e0.tail:
e0 = copy.deepcopy(e0)
e0.tail = ''
return [e0]
else:
potential_paragraphs = self.parser.getElementsByTag(
current_sibling, tag='p')
if potential_paragraphs is None:
return None
else:
ps = []
for first_paragraph in potential_paragraphs:
text = self.parser.getText(first_paragraph)
if len(text) > 0:
word_stats = self.stopwords_class(
language=self.language). \
get_stopword_count(text)
paragraph_score = word_stats.get_stopword_count()
sibling_baseline_score = float(.30)
high_link_density = self.is_highlink_density(
first_paragraph)
score = float(baseline_score_siblings_para *
sibling_baseline_score)
if score < paragraph_score and not high_link_density:
p = self.parser.createElement(
tag='p', text=text, tail=None)
ps.append(p)
return ps
def get_siblings_score(self, top_node):
"""We could have long articles that have tons of paragraphs
so if we tried to calculate the base score against
the total text score of those paragraphs it would be unfair.
So we need to normalize the score based on the average scoring
of the paragraphs within the top node.
For example if our total score of 10 paragraphs was 1000
but each had an average value of 100 then 100 should be our base.
"""
base = 100000
paragraphs_number = 0
paragraphs_score = 0
nodes_to_check = self.parser.getElementsByTag(top_node, tag='p')
for node in nodes_to_check:
text_node = self.parser.getText(node)
word_stats = self.stopwords_class(language=self.language). \
get_stopword_count(text_node)
high_link_density = self.is_highlink_density(node)
if word_stats.get_stopword_count() > 2 and not high_link_density:
paragraphs_number += 1
paragraphs_score += word_stats.get_stopword_count()
if paragraphs_number > 0:
base = paragraphs_score / paragraphs_number
return base
def update_score(self, node, add_to_score):
"""Adds a score to the gravityScore Attribute we put on divs
we'll get the current score then add the score we're passing
in to the current.
"""
current_score = 0
score_string = self.parser.getAttribute(node, 'gravityScore')
if score_string:
current_score = float(score_string)
new_score = current_score + add_to_score
self.parser.setAttribute(node, "gravityScore", str(new_score))
def update_node_count(self, node, add_to_count):
"""Stores how many decent nodes are under a parent node
"""
current_score = 0
count_string = self.parser.getAttribute(node, 'gravityNodes')
if count_string:
current_score = int(count_string)
new_score = current_score + add_to_count
self.parser.setAttribute(node, "gravityNodes", str(new_score))
def is_highlink_density(self, e):
"""Checks the density of links within a node, if there is a high
link to text ratio, then the text is less likely to be relevant
"""
links = self.parser.getElementsByTag(e, tag='a')
if not links:
return False
text = self.parser.getText(e)
words = [word for word in text.split() if word.isalnum()]
if not words:
return True
words_number = float(len(words))
sb = []
for link in links:
sb.append(self.parser.getText(link))
link_text = ''.join(sb)
link_words = link_text.split()
num_link_words = float(len(link_words))
num_links = float(len(links))
link_divisor = float(num_link_words / words_number)
score = float(link_divisor * num_links)
if score >= 1.0:
return True
return False
# return True if score > 1.0 else False
def get_score(self, node):
"""Returns the gravityScore as an integer from this node
"""
return self.get_node_gravity_score(node) or 0
def get_node_gravity_score(self, node):
gravity_score = self.parser.getAttribute(node, 'gravityScore')
if not gravity_score:
return None
return float(gravity_score)
def nodes_to_check(self, doc):
"""Returns a list of nodes we want to search
on like paragraphs and tables
"""
nodes_to_check = []
for tag in ['p', 'pre', 'td']:
items = self.parser.getElementsByTag(doc, tag=tag)
nodes_to_check += items
return nodes_to_check
def is_table_and_no_para_exist(self, e):
sub_paragraphs = self.parser.getElementsByTag(e, tag='p')
for p in sub_paragraphs:
txt = self.parser.getText(p)
if len(txt) < 25:
self.parser.remove(p)
sub_paragraphs_2 = self.parser.getElementsByTag(e, tag='p')
if len(sub_paragraphs_2) == 0 and e.tag != "td":
return True
return False
def is_nodescore_threshold_met(self, node, e):
top_node_score = self.get_score(node)
current_node_score = self.get_score(e)
threshold = float(top_node_score * .08)
if (current_node_score < threshold) and e.tag != 'td':
return False
return True
def post_cleanup(self, top_node):
"""Remove any divs that looks like non-content, clusters of links,
or paras with no gusto; add adjacent nodes which look contenty
"""
node = self.add_siblings(top_node)
for e in self.parser.getChildren(node):
e_tag = self.parser.getTag(e)
if e_tag != 'p':
if self.is_highlink_density(e):
self.parser.remove(e)
return node