netkeibaからレース日程とレース一覧を取得する方法【第1回】pythonスクレイピングの基本編

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager

def get_driver(opt=1):
    options = Options()
    if opt is None:
        options.add_argument('--headless')
    return webdriver.Chrome(ChromeDriverManager().install(), options=options)

### ※最新のseleniumではwebdriver_manager不要でwebdriver.Chrome()だけで処理ができるみたいです。

driver = get_driver() # 実行

from time import sleep
from urllib import parse
from selenium.webdriver.common.by import By

def parse_url(url, search_s):
    parsed_url = parse.urlparse(url)
    params = parse.parse_qs(parsed_url.query)
    target = params.get(search_s, [None])[0]
    return target

def get_kaisai_date(driver, year, month):
    url = f"/calendar.html?year={year}&month={month}" # 禁止ワードらしいのでネットケイバのURLを入れてください
    driver.get(url)
    sleep(2)
    CalendarSelectMenu = driver.find_element(by=By.CLASS_NAME, value="CalendarSelectMenu")
    Race_Calendar_Main = CalendarSelectMenu.find_element(by=By.CLASS_NAME, value="Race_Calendar_Main")
    RaceCellBoxes = Race_Calendar_Main.find_elements(by=By.CLASS_NAME, value="RaceCellBox")
    ancs = []
    for RaceCellBox in RaceCellBoxes:
        _a = RaceCellBox.find_elements(by=By.TAG_NAME, value="a")
        if len(_a) == 1:
            ancs.append(_a[0].get_attribute("href"))
    kaisai_dates = [parse_url(anc, "kaisai_date") for anc in ancs]
    return kaisai_dates

years = ["2021", "2022"]
monthes = [str(m) for m in range(1, 13)]

kaisai_dates = []

for year in years:
    for month in monthes:
        kaisai_dates += get_kaisai_date(driver, year, month)
print(kaisai_dates)

from time import sleep
import pandas as pd
from selenium.webdriver.common.by import By

def get_race_list(driver, select_datetime):
    url = f'/top/race_list.html?kaisai_date={select_datetime}' # 禁止ワードらしいのでホームページを追加してください
    driver.get(url)
    sleep(3)
    # sleep(3)
    rs = driver.find_elements(by=By.CLASS_NAME, value='RaceList_DataList')
    rsnames = [rsn.find_element(by=By.CLASS_NAME, value='RaceList_DataTitle').text for rsn in rs ]
    racelist = []
    for rss in rs:
        rsname = rss.find_element(by=By.CLASS_NAME, value='RaceList_DataTitle').text
        rsis = rss.find_elements(by=By.CLASS_NAME, value='RaceList_DataItem')
        for rsi in rsis:
            rsia = rsi.find_element(by=By.TAG_NAME, value='a')
            rsin = rsi.find_element(by=By.CLASS_NAME, value='Race_Num')
            rsi2 = rsi.find_element(by=By.CLASS_NAME, value='RaceList_ItemContent')
            rsi2t = rsi2.find_element(by=By.CLASS_NAME, value='ItemTitle')
            rsi2gradebk = rsi2.find_elements(by=By.CLASS_NAME, value='Icon_GradeType')
            if len(rsi2gradebk) > 0 and 'Icon_GradeType ' in rsi2gradebk[0].get_attribute('class'):
                classes = rsi2gradebk[0].get_attribute('class').split(" ")
                rsi2grade = class_grade_mapper[classes[1]]
            else:
                rsi2grade = "未勝利"
            rsi2d = rsi2.find_element(by=By.CLASS_NAME, value='RaceData')
            rsi2d2 = rsi2d.find_element(by=By.CLASS_NAME, value='RaceList_Itemtime')
            rsi2d3 = rsi2d.find_elements(by=By.CLASS_NAME, value='RaceList_ItemLong')
            if rsi2d3 is not None and rsi2d3 != []:
                rsi2d3 = rsi2d3[0].text
            else:
                rsi2d3 = rsi2d.find_elements(by=By.TAG_NAME, value='span')[1].text
            rurl = rsia.get_attribute('href')
            rid = select_rid(rurl)
            racelist.append((rsname,rsin.text,rsi2t.text,rsi2d2.text,rsi2d3,rurl,rid,rsi2grade))
    racelistpd = pd.DataFrame(racelist,columns=['会場','レースno','タイトル','発走時間','距離タイプ','URL','RID','重賞'])
    racelistpd['回数'] = racelistpd['会場'].str[0]
    racelistpd['日目'] = racelistpd['会場'].str[6]
    racelistpd['会場'] = racelistpd['会場'].str[3:5]
    racelistpd['タイプ'] = racelistpd['距離タイプ'].str[0]
    racelistpd['距離'] = racelistpd['距離タイプ'].str[1:5]
    return racelistpd