airflow-dags/dags/load_osr.py

78 lines
3.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

from airflow import DAG
from airflow.decorators import task
from airflow.models import Variable
from airflow.providers.postgres.hooks.postgres import PostgresHook
import requests
import json
from requests.auth import HTTPBasicAuth
import datetime
from sqlalchemy import create_engine, text
import pandas as pd
from smbclient import register_session, scandir, open_file, remove, rename
import re
with DAG(
dag_id="smb_to_postgres",
start_date=datetime.datetime(2026, 1, 1),
schedule="@daily",
catchup=False,
tags=["smb", "postgres"],
) as dag:
@task
def process_files():
smb_config = Variable.get(
"smb_connection",
deserialize_json=True,
)
register_session(
smb_config['server'],
username=smb_config['username'],
password=smb_config['password'])
path = fr"\\{SERVER}\{smb_config['folder']}"
dfs = []
for region in scandir(path):
if region.is_dir() and region.name != '#recycle':
for entry in scandir(fr"{path}\{region.name}"):
if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")):
with open_file(entry.path, mode='rb') as f:
sheets = pd.read_excel(f, sheet_name= None)
for sheet in sheets.keys():
df = sheets[sheet]
stroika = df['Unnamed: 1'].iloc[2]
obj = df['Unnamed: 1'].iloc[5]
df = df.iloc[27:]
final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0]
df = df.loc[:final_ind-1]
df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ]
cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты']
df.columns = cols
df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost')
df['naimenovanie_stroiki'] = stroika
df['naimenovanie_obekta'] = obj
df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями'
df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date()
df['subekt_rf'] = region.name
dfs.append(df)
df_full = pd.concat(dfs)
hook = PostgresHook(
postgres_conn_id="postgres_default")
df_full.to_sql(
name="smb_osr",
schema="smb",
con=hook.get_sqlalchemy_engine(),
if_exists="append",
index=False,
chunksize=1000,
method="multi",
)
process_files()