Обновить dags/load_osr.py
This commit is contained in:
parent
c487e7d6c2
commit
1277f9132f
|
|
@ -0,0 +1,78 @@
|
||||||
|
from airflow import DAG
|
||||||
|
from airflow.decorators import task
|
||||||
|
from airflow.models import Variable
|
||||||
|
from airflow.providers.postgres.hooks.postgres import PostgresHook
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
from requests.auth import HTTPBasicAuth
|
||||||
|
import datetime
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
import pandas as pd
|
||||||
|
from smbclient import register_session, scandir, open_file, remove, rename
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="smb_to_postgres",
|
||||||
|
start_date=datetime.datetime(2026, 1, 1),
|
||||||
|
schedule="@daily",
|
||||||
|
catchup=False,
|
||||||
|
tags=["smb", "postgres"],
|
||||||
|
) as dag:
|
||||||
|
|
||||||
|
@task
|
||||||
|
def process_files():
|
||||||
|
smb_config = Variable.get(
|
||||||
|
"smb_connection",
|
||||||
|
deserialize_json=True,
|
||||||
|
)
|
||||||
|
register_session(
|
||||||
|
smb_config['server'],
|
||||||
|
username=smb_config['username'],
|
||||||
|
password=smb_config['password'])
|
||||||
|
|
||||||
|
path = fr"\\{SERVER}\{smb_config['folder']}"
|
||||||
|
|
||||||
|
dfs = []
|
||||||
|
for region in scandir(path):
|
||||||
|
if region.is_dir() and region.name != '#recycle':
|
||||||
|
for entry in scandir(fr"{path}\{region.name}"):
|
||||||
|
if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")):
|
||||||
|
with open_file(entry.path, mode='rb') as f:
|
||||||
|
sheets = pd.read_excel(f, sheet_name= None)
|
||||||
|
for sheet in sheets.keys():
|
||||||
|
df = sheets[sheet]
|
||||||
|
stroika = df['Unnamed: 1'].iloc[2]
|
||||||
|
obj = df['Unnamed: 1'].iloc[5]
|
||||||
|
df = df.iloc[27:]
|
||||||
|
final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0]
|
||||||
|
df = df.loc[:final_ind-1]
|
||||||
|
df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ]
|
||||||
|
cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты']
|
||||||
|
df.columns = cols
|
||||||
|
df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost')
|
||||||
|
df['naimenovanie_stroiki'] = stroika
|
||||||
|
df['naimenovanie_obekta'] = obj
|
||||||
|
df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями'
|
||||||
|
df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date()
|
||||||
|
df['subekt_rf'] = region.name
|
||||||
|
dfs.append(df)
|
||||||
|
|
||||||
|
df_full = pd.concat(dfs)
|
||||||
|
|
||||||
|
|
||||||
|
hook = PostgresHook(
|
||||||
|
postgres_conn_id="postgres_default")
|
||||||
|
|
||||||
|
df_full.to_sql(
|
||||||
|
name="smb_osr",
|
||||||
|
schema="smb",
|
||||||
|
con=hook.get_sqlalchemy_engine(),
|
||||||
|
if_exists="append",
|
||||||
|
index=False,
|
||||||
|
chunksize=1000,
|
||||||
|
method="multi",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
process_files()
|
||||||
Loading…
Reference in New Issue