78 lines
3.2 KiB
Python
78 lines
3.2 KiB
Python
from airflow import DAG
|
||
from airflow.decorators import task
|
||
from airflow.models import Variable
|
||
from airflow.providers.postgres.hooks.postgres import PostgresHook
|
||
import requests
|
||
import json
|
||
from requests.auth import HTTPBasicAuth
|
||
import datetime
|
||
from sqlalchemy import create_engine, text
|
||
import pandas as pd
|
||
from smbclient import register_session, scandir, open_file, remove, rename
|
||
import re
|
||
|
||
|
||
with DAG(
|
||
dag_id="smb_to_postgres",
|
||
start_date=datetime.datetime(2026, 1, 1),
|
||
schedule="@daily",
|
||
catchup=False,
|
||
tags=["smb", "postgres"],
|
||
) as dag:
|
||
|
||
@task
|
||
def process_files():
|
||
smb_config = Variable.get(
|
||
"smb_connection",
|
||
deserialize_json=True,
|
||
)
|
||
register_session(
|
||
smb_config['server'],
|
||
username=smb_config['username'],
|
||
password=smb_config['password'])
|
||
|
||
path = fr"\\{smb_config['server']}\{smb_config['folder']}"
|
||
|
||
dfs = []
|
||
for region in scandir(path):
|
||
if region.is_dir() and region.name != '#recycle':
|
||
for entry in scandir(fr"{path}\{region.name}"):
|
||
if entry.is_file() and entry.name.startswith("ОСР") and entry.name.startswith("ОСР") and entry.name.endswith((".xlsx", ".xls")):
|
||
with open_file(entry.path, mode='rb') as f:
|
||
sheets = pd.read_excel(f, sheet_name= None)
|
||
for sheet in sheets.keys():
|
||
df = sheets[sheet]
|
||
stroika = df['Unnamed: 1'].iloc[2]
|
||
obj = df['Unnamed: 1'].iloc[5]
|
||
df = df.iloc[27:]
|
||
final_ind = df['Unnamed: 0'][df['Unnamed: 0'].isna()].index[0]
|
||
df = df.loc[:final_ind-1]
|
||
df = df[['Unnamed: 2', 'Unnamed: 3','Unnamed: 4', 'Unnamed: 5', 'Unnamed: 6'] ]
|
||
cols = ['naimenovanie_lsr', 'Строительные работы', 'Монтажные работы', 'Оборудование', 'Прочие затраты']
|
||
df.columns = cols
|
||
df = pd.melt(df, id_vars=cols[0], value_vars=cols[1:], var_name='detalizatsiya_raschetov', value_name='smetnaya_stoimost')
|
||
df['naimenovanie_stroiki'] = stroika
|
||
df['naimenovanie_obekta'] = obj
|
||
df['vid_smeti'] = 'Эталонная смета' if '_Э' in entry.name else 'Смета с изменениями'
|
||
df['data'] = datetime.datetime.strptime(entry.name.split('.')[0].split('_')[1], "%Y%m%d").date()
|
||
df['subekt_rf'] = region.name
|
||
dfs.append(df)
|
||
|
||
df_full = pd.concat(dfs)
|
||
|
||
|
||
hook = PostgresHook(
|
||
postgres_conn_id="postgres_default")
|
||
|
||
df_full.to_sql(
|
||
name="smb_osr",
|
||
schema="smb",
|
||
con=hook.get_sqlalchemy_engine(),
|
||
if_exists="append",
|
||
index=False,
|
||
chunksize=1000,
|
||
method="multi",
|
||
)
|
||
|
||
|
||
process_files() |