Обновить dags/load_osr.py
This commit is contained in:
parent
c487e7d6c2
commit
1277f9132f
|
|
@ -0,0 +1,78 @@
|
|||
from airflow import DAG
|
||||
from airflow.decorators import task
|
||||
from airflow.models import Variable
|
||||
from airflow.providers.postgres.hooks.postgres import PostgresHook
|
||||
import requests
|
||||
import json
|
||||
from requests.auth import HTTPBasicAuth
|
||||
import datetime
|
||||
from sqlalchemy import create_engine, text
|
||||
import pandas as pd
|
||||
from smbclient import register_session, scandir, open_file, remove, rename
|
||||
import re
|
||||
|
||||
|
||||
with DAG(
|
||||
dag_id="smb_to_postgres",
|
||||
start_date=datetime.datetime(2026, 1, 1),
|
||||
schedule="@daily",
|
||||
catchup=False,
|
||||
tags=["smb", "postgres"],
|
||||
) as dag:
|
||||
|
||||
@task
|
||||
def process_files():
|
||||
smb_config = Variable.get(
|
||||
"smb_connection",
|
||||
deserialize_json=True,
|
||||
)
|
||||
register_session(
|
||||
smb_config['server'],
|
||||
username=smb_config['username'],
|
||||
password=smb_config['password'])
|
||||
|
||||
path = fr"\\{SERVER}\{smb_config['folder']}"
|
||||
|
||||
dfs = []
|
||||
for region in scandir(path):
|
||||
if region.is_dir() and region.name != '#recycle':
|
||||
for entry in scandir(fr"{path}\{region.name}"):
|
||||
if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")):
|
||||
with open_file(entry.path, mode='rb') as f:
|
||||
sheets = pd.read_excel(f, sheet_name= None)
|
||||
for sheet in sheets.keys():
|
||||
df = sheets[sheet]
|
||||
stroika = df['Unnamed: 1'].iloc[2]
|
||||
obj = df['Unnamed: 1'].iloc[5]
|
||||
df = df.iloc[27:]
|
||||
final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0]
|
||||
df = df.loc[:final_ind-1]
|
||||
df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ]
|
||||
cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты']
|
||||
df.columns = cols
|
||||
df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost')
|
||||
df['naimenovanie_stroiki'] = stroika
|
||||
df['naimenovanie_obekta'] = obj
|
||||
df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями'
|
||||
df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date()
|
||||
df['subekt_rf'] = region.name
|
||||
dfs.append(df)
|
||||
|
||||
df_full = pd.concat(dfs)
|
||||
|
||||
|
||||
hook = PostgresHook(
|
||||
postgres_conn_id="postgres_default")
|
||||
|
||||
df_full.to_sql(
|
||||
name="smb_osr",
|
||||
schema="smb",
|
||||
con=hook.get_sqlalchemy_engine(),
|
||||
if_exists="append",
|
||||
index=False,
|
||||
chunksize=1000,
|
||||
method="multi",
|
||||
)
|
||||
|
||||
|
||||
process_files()
|
||||
Loading…
Reference in New Issue