from airflow import DAG from airflow.decorators import task from airflow.models import Variable from airflow.providers.postgres.hooks.postgres import PostgresHook import requests import json from requests.auth import HTTPBasicAuth import datetime from sqlalchemy import create_engine, text import pandas as pd from smbclient import register_session, scandir, open_file, remove, rename import re with DAG( dag_id="smb_to_postgres", start_date=datetime.datetime(2026, 1, 1), schedule="@daily", catchup=False, tags=["smb", "postgres"], ) as dag: @task def process_files(): smb_config = Variable.get( "smb_connection", deserialize_json=True, ) register_session( smb_config['server'], username=smb_config['username'], password=smb_config['password']) path = fr"\\{SERVER}\{smb_config['folder']}" dfs = [] for region in scandir(path): if region.is_dir() and region.name != '#recycle': for entry in scandir(fr"{path}\{region.name}"): if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")): with open_file(entry.path, mode='rb') as f: sheets = pd.read_excel(f, sheet_name= None) for sheet in sheets.keys(): df = sheets[sheet] stroika = df['Unnamed: 1'].iloc[2] obj = df['Unnamed: 1'].iloc[5] df = df.iloc[27:] final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0] df = df.loc[:final_ind-1] df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ] cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты'] df.columns = cols df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost') df['naimenovanie_stroiki'] = stroika df['naimenovanie_obekta'] = obj df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями' df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date() df['subekt_rf'] = region.name dfs.append(df) df_full = pd.concat(dfs) hook = PostgresHook( postgres_conn_id="postgres_default") df_full.to_sql( name="smb_osr", schema="smb", con=hook.get_sqlalchemy_engine(), if_exists="append", index=False, chunksize=1000, method="multi", ) process_files()