优化任务处理逻辑,添加限流机制以控制并发任务数量
All checks were successful
Sync Container Images / sync-images (push) Successful in 1m16s
All checks were successful
Sync Container Images / sync-images (push) Successful in 1m16s
This commit is contained in:
parent
cf6771f945
commit
30a1b7de6a
15
sync.py
15
sync.py
@ -1,7 +1,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiofiles
|
import aiofiles
|
||||||
import yaml
|
import yaml
|
||||||
from pydantic import BaseModel, field_validator
|
from pydantic import BaseModel
|
||||||
|
|
||||||
class Config(BaseModel):
|
class Config(BaseModel):
|
||||||
images: list["Images"] = []
|
images: list["Images"] = []
|
||||||
@ -39,6 +39,12 @@ async def docker_push(image: str) -> int | None:
|
|||||||
print(f"Pushing image {image}...")
|
print(f"Pushing image {image}...")
|
||||||
return await run_command(f"docker push {image}")
|
return await run_command(f"docker push {image}")
|
||||||
|
|
||||||
|
semaphore = asyncio.Semaphore(5)
|
||||||
|
|
||||||
|
async def limited_task(task):
|
||||||
|
async with semaphore:
|
||||||
|
return await task
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
async with aiofiles.open('images.yaml', 'r') as file:
|
async with aiofiles.open('images.yaml', 'r') as file:
|
||||||
config = await file.read()
|
config = await file.read()
|
||||||
@ -57,20 +63,21 @@ async def main():
|
|||||||
await docker_pull(f"{image.source}:{tag}")
|
await docker_pull(f"{image.source}:{tag}")
|
||||||
await docker_tag(f"{image.source}:{tag}", f"{image.target}:{tag}")
|
await docker_tag(f"{image.source}:{tag}", f"{image.target}:{tag}")
|
||||||
await docker_push(f"{image.target}:{tag}")
|
await docker_push(f"{image.target}:{tag}")
|
||||||
tasks.append(task())
|
tasks.append(limited_task(task()))
|
||||||
else:
|
else:
|
||||||
async def task():
|
async def task():
|
||||||
await docker_pull(image.source)
|
await docker_pull(image.source)
|
||||||
await docker_tag(image.source, image.target)
|
await docker_tag(image.source, image.target)
|
||||||
await docker_push(image.target)
|
await docker_push(image.target)
|
||||||
tasks.append(task())
|
tasks.append(limited_task(task()))
|
||||||
|
|
||||||
results = await asyncio.gather(*tasks)
|
results = await asyncio.gather(*tasks)
|
||||||
failed_tasks = 0
|
failed_tasks = 0
|
||||||
for result in results:
|
for result in results:
|
||||||
if result is not None and result != 0:
|
if result is not None and result != 0:
|
||||||
failed_tasks += 1
|
failed_tasks += 1
|
||||||
print(f"{len(results)} tasks completed. {len(result) - failed_tasks} succeed, {failed_tasks} failed.")
|
|
||||||
|
print(f"{len(results)} tasks completed. {len(results) - failed_tasks} succeed, {failed_tasks} failed.")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
Loading…
x
Reference in New Issue
Block a user