From 1277f9132fc5b10591c0cb17000ed38d68953ee6 Mon Sep 17 00:00:00 2001 From: bn_user Date: Fri, 10 Jul 2026 11:40:53 +0000 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20dags/load=5Fosr.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/load_osr.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/dags/load_osr.py b/dags/load_osr.py index e69de29..f7db75a 100644 --- a/dags/load_osr.py +++ b/dags/load_osr.py @@ -0,0 +1,78 @@ +from airflow import DAG +from airflow.decorators import task +from airflow.models import Variable +from airflow.providers.postgres.hooks.postgres import PostgresHook +import requests +import json +from requests.auth import HTTPBasicAuth +import datetime +from sqlalchemy import create_engine, text +import pandas as pd +from smbclient import register_session, scandir, open_file, remove, rename +import re + + +with DAG( + dag_id="smb_to_postgres", + start_date=datetime.datetime(2026, 1, 1), + schedule="@daily", + catchup=False, + tags=["smb", "postgres"], +) as dag: + + @task + def process_files(): + smb_config = Variable.get( + "smb_connection", + deserialize_json=True, + ) + register_session( + smb_config['server'], + username=smb_config['username'], + password=smb_config['password']) + + path = fr"\\{SERVER}\{smb_config['folder']}" + + dfs = [] + for region in scandir(path): + if region.is_dir() and region.name != '#recycle': + for entry in scandir(fr"{path}\{region.name}"): + if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")): + with open_file(entry.path, mode='rb') as f: + sheets = pd.read_excel(f, sheet_name= None) + for sheet in sheets.keys(): + df = sheets[sheet] + stroika = df['Unnamed: 1'].iloc[2] + obj = df['Unnamed: 1'].iloc[5] + df = df.iloc[27:] + final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0] + df = df.loc[:final_ind-1] + df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ] + cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты'] + df.columns = cols + df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost') + df['naimenovanie_stroiki'] = stroika + df['naimenovanie_obekta'] = obj + df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями' + df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date() + df['subekt_rf'] = region.name + dfs.append(df) + + df_full = pd.concat(dfs) + + + hook = PostgresHook( + postgres_conn_id="postgres_default") + + df_full.to_sql( + name="smb_osr", + schema="smb", + con=hook.get_sqlalchemy_engine(), + if_exists="append", + index=False, + chunksize=1000, + method="multi", + ) + + + process_files() \ No newline at end of file