beautifulsoup解析页面
from bs4 import BeautifulSoupsoup = BeautifulSoup(htmltxt, "lxml")# 三种装载器soup = BeautifulSoup("<a></p>", "html.parser")### 只有起始标签的会自动补全，只有结束标签的会自动忽略### 结果为：<a></a>soup = BeautifulSoup("<a></p>", "lxml")### 结果为：<html><body><a></a></body></html>soup = BeautifulSoup("<a></p>", "html5lib")### html5lib则出现一般的标签都会自动补全### 结果为：<html><head></head><body><a><p></p></a></body></html># 根据标签名、id、class、属性等查找标签### 根据class、id、以及属性alog-action的值和标签类别查询soup.find("a",class_="title",id="t1",attrs={"alog-action": "qb-ask-uname"}))### 查询标签内某属性的值pubtime = soup.find("meta",attrs={"itemprop":"datePublished"}).attrs['content']### 获取所有class为title的标签for i in soup.find_all(class_="title"): print(i.get_text())### 获取特定数量的class为title的标签for i in soup.find_all(class_="title",limit = 2): print(i.get_text())### 获取文本内容时可以指定不同标签之间的分隔符，也可以选择是否去掉前后的空白。soup = BeautifulSoup('<p class="title" id="p1"><b> The Dormouses story </b></p><p class="title" id="p1"><b>The Dormouses story</b></p>', "html5lib")soup.find(class_="title").get_text("|", strip=True)#结果为：The Dormouses story|The Dormouses story### 获取class为title的p标签的idsoup.find(class_="title").get("id")### 对class名称正则：soup.find_all(class_=re.compile("tit"))### recursive参数，recursive=False时，只find当前标签的第一级子标签的数据soup = BeautifulSoup('<html><head><title>abc','lxml')soup.html.find_all("title", recursive=False)unicode编码转中文
content = "/u65f6/u75c7/u5b85"content = content.encode("utf8","ignore").decode('unicode_escape')url encode的解码与解码
from urllib import parse# 编码x = "中国你好"y = parse.quote(x)print(y)# 解码x = parse.unquote(y)print(x)
html转义字符的解码
from html.parser import HTMLParserhtmls = "<div><p>"txt = HTMLParser().unescape(htmls)print(txt) . # 输出<div><p>
base64的编码与解码
import base64# 编码content = "测试转码文本123"contents_base64 = base64.b64encode(content.encode('utf-8','ignore')).decode("utf-8")# 解码contents = base64.b64decode(contents_base64)过滤emoji表情
def filter_emoji(desstr,restr=''): try: co = re.compile(u'[/U00010000-/U0010ffff]') except re.error: co = re.compile(u'[/uD800-/uDBFF][/uDC00-/uDFFF]') return co.sub(restr, desstr)
新闻热点
疑难解答
