python全栈数据工程师养成攻略 #

def print_hi(name):
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.
if __name__ == '__main__':
    print_hi('PyCharm')

def python02():
    # 数字
    a = 1
    b = 2.1
    print(a,b)
    # 字符串
    c = "Hello"
    d = "你好"
    print(c,d)
    print(c+d)
    print(len(c+d))
    # 位置下标从0开始
    c1 = "Hello World"
    print(c1[0],c1[-1],c[1:5],c[1:-1],c[:5],c[3:])
    # 列表 相当于数组
    arr = []
    arr.append(1)
    arr.append(2.1)
    arr.append('Hello')
    print(arr,len(arr),arr[1],arr[-1])
    arr[1] = 100
    print(arr)
    del arr[0]
    print(arr)
    # 元组
    arr2 = (1,2.1,'Hello')
    # arr2[0] = 22
    print(arr2)
    # 字典 相当于对象
    aobj = {}
    aobj['k1'] = 1
    aobj['k2'] = 2.1
    aobj['k3'] = 'Hello'
    print(aobj)
    li = [1,2,'hell']
    di = {'k1':1,'k2':2.1,'k3':'hell'}
    print(di.get('k1'))
    '''
        注释
    '''
    a2 = 1
    b2 = 2
    print(a2+b2)
    print(a2 == b2)
    c2 = True
    d2 = False
    print(c2 and d2, c2 or d2, not c2)
    a3 = 1
    # 单个条件
    if a3 == 1:
        print(1111)
    # 处理条件不成立
    if a3 == 2:
        print(222)
    else:
        print(3333)
    # 多个条件,加多少个都可以
    if a3 == 1:
        print(111)
    elif a3 == 2:
        print(2222)
    else:
        print(33333)

    # 循环
    flag = 1
    while flag < 10:
        # while一定要改变条件变量 不然死循环
        print(flag)
        flag += 1
    # x从0开始 一直9结束 即不包括后面的数字
    for x in range(0,10):
        print(x)
    # 遍历数组
    for item in li:
        print(item)
    # 遍历对象的键
    for key in di.keys():
        print(key)
    # 遍历对象的值
    for value in di.values():
        print(value)
    # 同时遍历对象的键和值
    for key,value in di.items():
        print(key,value)
    # 循环控制
    for x in range(0,10):
        if x == 5:
            pass
        else:
            print(x)
    for x in range(0,10):
        if x ==5:
            continue
        print(x)
    for x in range(0,10):
        if x == 3:
            break
        print(x)

    # 时间
    import time
    t = time.time()
    print(t,type(t))
    t1 = '2023-07-06 10:00:00'
    t2 = int(time.mktime(time.strptime(t1,"%Y-%m-%d %H:%M:%S")))
    print(t2)
    t3 = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(t))
    print(t3)
if __name__ == '__main__':
    python02()

def python03():
    fw = open('./data.txt','w')
    for x in range(0,10):
        fw.write(str(x))
    fw.close()
    file = open('./data.txt','r')
    while True:  # 循环
        line = file.readline()  # 读取一行内容存入变量
        if line:  # 如果变量不是空值（末尾行为空值）
            print(line, end='')  # 显示输出读取结果，设置不换行输出（文本的每一行都带有换行符）。
        else:  # 否则
            break  # 结束循环
    file.close()  # 关闭文件
    try:
        print(1/0)
    except Exception as e:
        print(e)
    finally:
        print("一定会执行")
if __name__ == '__main__':
    python03()

def python_xyj():
    file = open('./xyj.txt','r',encoding='utf-8')
    # 出现的汉字
    characters = []
    # 每个汉字出现的次数
    stat = {}
    while True:  # 循环
        line = file.readline()  # 读取一行内容存入变量
        if line:  # 如果变量不是空值（末尾行为空值）
            # 去掉每一行两边的空白
            line = line.strip()
            if len(line) == 0:
                continue
            for x in range(0,len(line)):
                if line[x] in [' ', '\t', '\n', '。', '，', '(', ')', '（', '）', '：', '□', '？', '！', '《', '》', '、', '；','“', '”', '……']:
                    continue
                if not line[x] in characters:
                    characters.append(line[x])
                if not stat.get(line[x]):
                    stat[line[x]] = 0

                stat[line[x]] += 1
        else:  # 否则
            break  # 结束循环
    file.close()  # 关闭文件
if __name__ == '__main__':
    python_xyj()

import requests
from requests.exceptions import ReadTimeout
import json

def python_featch():
    # 掘金前端的api
    url = "https://api.juejin.cn/content_api/v1/content/article_rank?category_id=6809637767543259144&type=hot"
    response = requests.get(url)
    print(response.text)
    print(response.json())				# 使用request的内置json API
    print(json.loads(response.text))	# 使用json库的api

    # 超时设置 超过0.5s会返回Timeout
    try:
        response = requests.get("http://httpbin.org/get", timeout=0.5)
        print(response.status_code)
    except ReadTimeout:
        print('Timeout')
    # post 百度指数
    response2 = requests.post("https://index.baidu.com/Interface/homePage/pcConfig")
    print(response2.json())

# 获取豆瓣电影数据并存入txt中
def python_douban():
    # 电影标签  https://movie.douban.com/j/search_tags?type=movie&source=index
    # 电影列表 https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&page_limit=50&page_start=0
    # 电影详情 https://movie.douban.com/subject/35401290/
    url_tag = 'https://movie.douban.com/j/search_tags?type=movie&source=index'
    tags = []
    # 避免返回<response 400>
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.68'
    }

    try:
        response_tag = requests.get(url_tag,headers=headers).json()
        print(response_tag['tags'])
        tags = response_tag['tags']

    except ReadTimeout:
        print('Timeout')

    # 获取电影
    movies = []
    for tag in tags:
        start = 0
        # 不断请求,直到结果返回空
        while True:
            url_movies = "https://movie.douban.com/j/search_subjects?type=movie&tag="+ tag +"&page_limit=20&page_start="+str(start)
            print(url_movies)
            try:
                response_movies = requests.get(url_movies, headers=headers).json()
                print(response_movies['subjects'])
                response_movies_data = response_movies['subjects']
                if len(response_movies_data) == 0:
                    break
                for item in response_movies_data:
                    movies.append(item)
            except ReadTimeout:
                print('Timeout')



            # 新加20
            start += 20
    print(len(movies))
    # 获取电影详情页
    python_douban_movie_detail(movies)
    #将结果写入txt中
    fw = open('douban_movies.txt','w',encoding='utf-8')
    #写入一行表头,用于说明每个字段的意义
    print("创建文件douban_movies.txt,并录入数据")

    fw.write('title^rate^url^cover^id^description\n')
    for item in movies:
        movie_info = item['title'] + '^'+item['rate'] + '^'+item['url']+'^'+item['cover']+'^'+item['id']+'^'+item.get('description')+'\n'
        fw.write(movie_info)
    fw.close()
    print("操作结束!已爬取到豆瓣电影数据")
    
# 获取豆瓣电影的电影详情页的电影介绍数据
def python_douban_movie_detail(movies):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.68'
    }
    for x in range(0, len(movies)):
        # print(movies[x])
        print("录入第",x)
        # 页面爬取数据过慢 只录取前10条
        if x == 10:
            break
        url = movies[x]['url']
        request_movie_detail = requests.get(url, headers=headers).text
        html = BeautifulSoup(request_movie_detail, "html.parser")  # 指定解析器

        try:
            detail = html.find_all("span", attrs={"property": "v:summary"})[0].get_text()
            # print(detail)
        except Exception as e:
            movies[x]['description'] = ''

        else:
            movies[x]['description'] = detail
        finally:
            pass
        time.sleep(0.5) # 0.5s请求一次 避免请求过快 无法响应
    # print(movies)

if __name__ == '__main__':
	python_douban()