Although I couldn't find a 'gsutil' command to empty a bucket as such the following will remove and then we add the bucket.
I've also add the code here for how to do this from an Airflow DAG .
% gsutil -m rm -r gs://djem_photos
% gsutil -m mkdir gs://djem_photos
from airflow.operators import bash_operator
bucket = 'gs://temp_gp_tasks'
with models.DAG(
'remove_files_from_gcs',
schedule_interval=None,
default_args=default_dag_args,
catchup=True) as dag:
remove_gcs_bucket = bash_operator.BashOperator(
task_id='remove_gcs_bucket',
bash_command='gsutil -m rm -r {bucket}'.format(
bucket=bucket))
recreate_gcs_bucket = bash_operator.BashOperator(
task_id='recreate_gcs_bucket',
bash_command='gsutil -m mkdir {bucket}'.format(
bucket=bucket))
remove_gcs_bucket >> recreate_gcs_bucket
No comments:
Post a Comment