Обновить dags/load_osr.py

This commit is contained in:
bn_user 2026-07-10 11:40:53 +00:00
parent c487e7d6c2
commit 1277f9132f
1 changed files with 78 additions and 0 deletions

View File

@ -0,0 +1,78 @@
from airflow import DAG
from airflow.decorators import task
from airflow.models import Variable
from airflow.providers.postgres.hooks.postgres import PostgresHook
import requests
import json
from requests.auth import HTTPBasicAuth
import datetime
from sqlalchemy import create_engine, text
import pandas as pd
from smbclient import register_session, scandir, open_file, remove, rename
import re
with DAG(
dag_id="smb_to_postgres",
start_date=datetime.datetime(2026, 1, 1),
schedule="@daily",
catchup=False,
tags=["smb", "postgres"],
) as dag:
@task
def process_files():
smb_config = Variable.get(
"smb_connection",
deserialize_json=True,
)
register_session(
smb_config['server'],
username=smb_config['username'],
password=smb_config['password'])
path = fr"\\{SERVER}\{smb_config['folder']}"
dfs = []
for region in scandir(path):
if region.is_dir() and region.name != '#recycle':
for entry in scandir(fr"{path}\{region.name}"):
if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")):
with open_file(entry.path, mode='rb') as f:
sheets = pd.read_excel(f, sheet_name= None)
for sheet in sheets.keys():
df = sheets[sheet]
stroika = df['Unnamed: 1'].iloc[2]
obj = df['Unnamed: 1'].iloc[5]
df = df.iloc[27:]
final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0]
df = df.loc[:final_ind-1]
df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ]
cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты']
df.columns = cols
df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost')
df['naimenovanie_stroiki'] = stroika
df['naimenovanie_obekta'] = obj
df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями'
df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date()
df['subekt_rf'] = region.name
dfs.append(df)
df_full = pd.concat(dfs)
hook = PostgresHook(
postgres_conn_id="postgres_default")
df_full.to_sql(
name="smb_osr",
schema="smb",
con=hook.get_sqlalchemy_engine(),
if_exists="append",
index=False,
chunksize=1000,
method="multi",
)
process_files()