python 正文内容提取
#coding:utf-8import redef remove_js_css (content): """ remove the the javascript and the stylesheet and the comment content (<script>....</script> and <style>....</style> <!-- xxx -->) """ r = re.compile(r'''<script.*?</script>''',re.I|re.M|re.S) s = r.sub ('',content) r = re.compile(r'''<style.*?</style>''',re.I|re.M|re.S) s = r.sub ('', s) r = re.compile(r'''<!--.*?-->''', re.I|re.M|re.S) s = r.sub('',s) r = re.compile(r'''<meta.*?>''', re.I|re.M|re.S) s = r.sub('',s) r = re.compile(r'''<ins.*?</ins>''', re.I|re.M|re.S) s = r.sub('',s) return sdef remove_empty_line (content): """remove multi space """ r = re.compile(r'''^\s+$''', re.M|re.S) s = r.sub ('', content) r = re.compile(r'''\n+''',re.M|re.S) s = r.sub('\n',s) return sdef remove_any_tag (s): s = re.sub(r'''<[^>]+>''','',s) return s.strip()def remove_any_tag_but_a (s): text = re.findall (r'''<a[^r][^>]*>(.*?)</a>''',s,re.I|re.S|re.S) text_b = remove_any_tag (s) return len(''.join(text)),len(text_b)def remove_image (s,n=50): image = 'a' * n r = re.compile (r'''<img.*?>''',re.I|re.M|re.S) s = r.sub(image,s) return sdef remove_video (s,n=1000): video = 'a' * n r = re.compile (r'''<embed.*?>''',re.I|re.M|re.S) s = r.sub(video,s) return sdef sum_max (values): cur_max = values[0] glo_max = -999999 left,right = 0,0 for index,value in enumerate (values): cur_max += value if (cur_max > glo_max) : glo_max = cur_max right = index elif (cur_max < 0): cur_max = 0 for i in range(right, -1, -1): glo_max -= values[i] if abs(glo_max < 0.00001): left = i break return left,right+1def method_1 (content, k=1): if not content: return None,None,None,None tmp = content.split('\n') group_value = [] for i in range(0,len(tmp),k): group = '\n'.join(tmp[i:i+k]) group = remove_image (group) group = remove_video (group) text_a,text_b= remove_any_tag_but_a (group) temp = (text_b - text_a) - 8 group_value.append (temp) left,right = sum_max (group_value) return left,right, len('\n'.join(tmp[:left])), len ('\n'.join(tmp[:right]))def extract (content): content = remove_empty_line(remove_js_css(content)) left,right,x,y = method_1 (content) return '\n'.join(content.split('\n')[left:right])
?
代码 从最后一个函数开始调用。
