知微
fa45d8aa5f
- health_checklist.json: 192.168.1.122→node122
- ocr_client.py: docstring IP→node122
- docs/market-data-requirements.md: IP→node122
- 所有API调用通过ProxyHandler({})绕过系统代理
Privoxy对node122:18003返回500，直连正常
247 lines
8.0 KiB
Python
247 lines
8.0 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding:utf-8 -*-
|
|
"""
|
|
Date: 2024/4/3 16:08
|
|
Desc: 金十数据-数据中心-主要机构-宏观经济
|
|
https://datacenter.jin10.com/
|
|
"""
|
|
|
|
import datetime
|
|
import time
|
|
|
|
import pandas as pd
|
|
import requests
|
|
from tqdm import tqdm
|
|
|
|
|
|
def macro_cons_gold() -> pd.DataFrame:
|
|
"""
|
|
全球最大黄金 ETF—SPDR Gold Trust 持仓报告, 数据区间从 20041118-至今
|
|
https://datacenter.jin10.com/reportType/dc_etf_gold
|
|
:return: 持仓报告
|
|
:rtype: pandas.DataFrame
|
|
"""
|
|
t = time.time()
|
|
headers = {
|
|
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/107.0.0.0 Safari/537.36",
|
|
"x-app-id": "rU6QIu7JHe2gOUeR",
|
|
"x-csrf-token": "x-csrf-token",
|
|
"x-version": "1.0.0",
|
|
}
|
|
url = "https://datacenter-api.jin10.com/reports/list_v2"
|
|
params = {
|
|
"category": "etf",
|
|
"attr_id": "1",
|
|
"max_date": "",
|
|
"_": str(int(round(t * 1000))),
|
|
}
|
|
big_df = pd.DataFrame()
|
|
while True:
|
|
r = requests.get(url, params=params, headers=headers)
|
|
data_json = r.json()
|
|
if not data_json["data"]["values"]:
|
|
break
|
|
temp_df = pd.DataFrame(data_json["data"]["values"])
|
|
big_df = pd.concat(objs=[big_df, temp_df], ignore_index=True)
|
|
last_date_str = temp_df.iat[-1, 0]
|
|
last_date_str = (
|
|
(
|
|
datetime.datetime.strptime(last_date_str, "%Y-%m-%d")
|
|
- datetime.timedelta(days=1)
|
|
)
|
|
.date()
|
|
.isoformat()
|
|
)
|
|
params.update({"max_date": f"{last_date_str}"})
|
|
big_df.columns = [
|
|
"日期",
|
|
"总库存",
|
|
"增持/减持",
|
|
"总价值",
|
|
]
|
|
big_df["商品"] = "黄金"
|
|
big_df = big_df[
|
|
[
|
|
"商品",
|
|
"日期",
|
|
"总库存",
|
|
"增持/减持",
|
|
"总价值",
|
|
]
|
|
]
|
|
big_df["日期"] = pd.to_datetime(big_df["日期"], errors="coerce").dt.date
|
|
big_df["总库存"] = pd.to_numeric(big_df["总库存"], errors="coerce")
|
|
big_df["增持/减持"] = pd.to_numeric(big_df["增持/减持"], errors="coerce")
|
|
big_df["总价值"] = pd.to_numeric(big_df["总价值"], errors="coerce")
|
|
big_df.sort_values(["日期"], inplace=True)
|
|
big_df.reset_index(inplace=True, drop=True)
|
|
return big_df
|
|
|
|
|
|
def macro_cons_silver() -> pd.DataFrame:
|
|
"""
|
|
全球最大白银 ETF—SPDR Gold Trust 持仓报告, 数据区间从 20041118-至今
|
|
https://datacenter.jin10.com/reportType/dc_etf_sliver
|
|
:return: 持仓报告
|
|
:rtype: pandas.DataFrame
|
|
"""
|
|
t = time.time()
|
|
headers = {
|
|
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/107.0.0.0 Safari/537.36",
|
|
"x-app-id": "rU6QIu7JHe2gOUeR",
|
|
"x-csrf-token": "x-csrf-token",
|
|
"x-version": "1.0.0",
|
|
}
|
|
url = "https://datacenter-api.jin10.com/reports/list_v2"
|
|
params = {
|
|
"category": "etf",
|
|
"attr_id": "2",
|
|
"max_date": "",
|
|
"_": str(int(round(t * 1000))),
|
|
}
|
|
big_df = pd.DataFrame()
|
|
while True:
|
|
r = requests.get(url, params=params, headers=headers)
|
|
data_json = r.json()
|
|
if not data_json["data"]["values"]:
|
|
break
|
|
temp_df = pd.DataFrame(data_json["data"]["values"])
|
|
big_df = pd.concat(objs=[big_df, temp_df], ignore_index=True)
|
|
last_date_str = temp_df.iat[-1, 0]
|
|
last_date_str = (
|
|
(
|
|
datetime.datetime.strptime(last_date_str, "%Y-%m-%d")
|
|
- datetime.timedelta(days=1)
|
|
)
|
|
.date()
|
|
.isoformat()
|
|
)
|
|
params.update({"max_date": f"{last_date_str}"})
|
|
big_df.columns = [
|
|
"日期",
|
|
"总库存",
|
|
"增持/减持",
|
|
"总价值",
|
|
]
|
|
big_df["商品"] = "白银"
|
|
big_df = big_df[
|
|
[
|
|
"商品",
|
|
"日期",
|
|
"总库存",
|
|
"增持/减持",
|
|
"总价值",
|
|
]
|
|
]
|
|
big_df["日期"] = pd.to_datetime(big_df["日期"], errors="coerce").dt.date
|
|
big_df["总库存"] = pd.to_numeric(big_df["总库存"], errors="coerce")
|
|
big_df["增持/减持"] = pd.to_numeric(big_df["增持/减持"], errors="coerce")
|
|
big_df["总价值"] = pd.to_numeric(big_df["总价值"], errors="coerce")
|
|
big_df.sort_values(["日期"], inplace=True)
|
|
big_df.reset_index(inplace=True, drop=True)
|
|
return big_df
|
|
|
|
|
|
def macro_cons_opec_month() -> pd.DataFrame:
|
|
"""
|
|
欧佩克报告-月度, 数据区间从 20170118-至今
|
|
这里返回的具体索引日期的数据为上一个月的数据, 由于某些国家的数据有缺失
|
|
只选择有数据的国家返回
|
|
20200312:fix:由于 “厄瓜多尔” 已经有几个月没有更新数据，在这里加以剔除
|
|
https://datacenter.jin10.com/reportType/dc_opec_report
|
|
:return: 欧佩克报告-月度
|
|
:rtype: pandas.DataFrame
|
|
"""
|
|
t = time.time()
|
|
big_df = pd.DataFrame()
|
|
headers = {
|
|
"accept": "*/*",
|
|
"accept-encoding": "gzip, deflate, br",
|
|
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"cache-control": "no-cache",
|
|
"origin": "https://datacenter.jin10.com",
|
|
"pragma": "no-cache",
|
|
"referer": "https://datacenter.jin10.com/reportType/dc_opec_report",
|
|
"sec-fetch-mode": "cors",
|
|
"sec-fetch-site": "same-site",
|
|
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/79.0.3945.117 Safari/537.36",
|
|
"x-app-id": "rU6QIu7JHe2gOUeR",
|
|
"x-csrf-token": "",
|
|
"x-version": "1.0.0",
|
|
}
|
|
res = requests.get(
|
|
url=f"https://datacenter-api.jin10.com/reports/dates?category=opec&_={str(int(round(t * 1000)))}",
|
|
headers=headers,
|
|
) # 日期序列
|
|
all_date_list = res.json()["data"]
|
|
bar = tqdm(reversed(all_date_list))
|
|
for item in bar:
|
|
bar.set_description(f"Please wait for a moment, now downloading {item}'s data")
|
|
res = requests.get(
|
|
url=f"https://datacenter-api.jin10.com/reports/list?"
|
|
f"category=opec&date={item}&_={str(int(round(t * 1000)))}",
|
|
headers=headers,
|
|
)
|
|
temp_df = pd.DataFrame(
|
|
res.json()["data"]["values"],
|
|
columns=pd.DataFrame(res.json()["data"]["keys"])["name"].tolist(),
|
|
).T
|
|
temp_df.columns = temp_df.iloc[0, :]
|
|
temp_df = temp_df.iloc[1:, :]
|
|
try:
|
|
temp_df = temp_df[
|
|
[
|
|
"阿尔及利亚",
|
|
"安哥拉",
|
|
"加蓬",
|
|
"伊朗",
|
|
"伊拉克",
|
|
"科威特",
|
|
"利比亚",
|
|
"尼日利亚",
|
|
"沙特",
|
|
"阿联酋",
|
|
"委内瑞拉",
|
|
"欧佩克产量",
|
|
]
|
|
].iloc[-2, :]
|
|
except: # noqa: E722
|
|
temp_df = temp_df[
|
|
[
|
|
"阿尔及利亚",
|
|
"安哥拉",
|
|
"加蓬",
|
|
"伊朗",
|
|
"伊拉克",
|
|
"科威特",
|
|
"利比亚",
|
|
"尼日利亚",
|
|
"沙特",
|
|
"阿联酋",
|
|
"委内瑞拉",
|
|
"欧佩克产量",
|
|
]
|
|
].iloc[-1, :]
|
|
temp_df.dropna(inplace=True)
|
|
big_df[temp_df.name] = temp_df
|
|
big_df = big_df.T
|
|
big_df = big_df.astype(float)
|
|
big_df.reset_index(inplace=True)
|
|
big_df.rename(columns={"index": "日期"}, inplace=True)
|
|
big_df.columns.name = None
|
|
return big_df
|
|
|
|
|
|
if __name__ == "__main__":
|
|
macro_cons_gold_df = macro_cons_gold()
|
|
print(macro_cons_gold_df)
|
|
|
|
macro_cons_silver_df = macro_cons_silver()
|
|
print(macro_cons_silver_df)
|
|
|
|
macro_cons_opec_month_df = macro_cons_opec_month()
|
|
print(macro_cons_opec_month_df)
|