46320

获取url

import requestsfrom bs4 import BeautifulSoupimport reclass Get_links(object): def __init__(self): self.url = 'http://www.jianfeiketang.com' self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3724.8 Safari/537.36", } def parse(self): response = requests.get(url=self.url, headers=self.headers) html = response.content.decode() links = [] soup = BeautifulSoup(html, "html.parser") url_list = soup.find_all('a') for link in url_list: links.append(link.get('href')) return links def save(self, links): for link in links: with open('../result/jf.txt', 'a', encoding='utf-8') as f: f.write(link + '\n') print('保存成功...') def guolv(self): with open('../result/jf.txt', 'r', encoding='utf-8') as f: lines = f.readlines() for line in lines: # 循环处理每行数据 urls = re.findall(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", line) for url in urls: with open('./jfresult.txt', 'a', encoding='utf-8') as f: f.write(url + '\n') print('清洗完毕。。。')if __name__ == '__main__': get_links = Get_links() links = get_links.parse() get_links.save(links) get_links.guolv()

来源:博客园

作者:Victor_JJ

链接:https://www.cnblogs.com/victorstudy/p/11425895.html

Recommend