Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
B
Base Worker
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Workers
Base Worker
Commits
18835268
Commit
18835268
authored
3 years ago
by
Valentin Rigal
Browse files
Options
Downloads
Patches
Plain Diff
Suggestion
parent
24766547
No related branches found
No related tags found
1 merge request
!109
Handle concurrency while initializing activity
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
arkindex_worker/worker/__init__.py
+21
-18
21 additions, 18 deletions
arkindex_worker/worker/__init__.py
with
21 additions
and
18 deletions
arkindex_worker/worker/__init__.py
+
21
−
18
View file @
18835268
...
...
@@ -127,23 +127,14 @@ class ElementsWorker(
logger
.
info
(
f
"
Processing
{
element
}
(
{
i
}
/
{
count
}
)
"
)
# Process the element and report its progress if activities are enabled
try
:
self
.
update_activity
(
element
.
id
,
ActivityState
.
Started
)
except
ErrorResponse
as
e
:
if
e
.
status_code
!=
409
:
raise
e
# 409 conflict error when updating the state of an activity to "started" mean that we
# cannot process this element. We assume that the reason is that the state transition
# was forbidden i.e. that the activity was already in a started or processed state.
# This allow concurrent access to an element activity between multiple processes.
# Element should not be counted as failed as it is probably handled somewhere else.
logger
.
warning
(
f
"
Cannot start processing element
{
element
.
id
}
due to a conflict.
"
f
"
Another process could have processed it with the same version already.
"
if
self
.
update_activity
(
element
.
id
,
ActivityState
.
Started
):
self
.
process_element
(
element
)
self
.
update_activity
(
element
.
id
,
ActivityState
.
Processed
)
else
:
logger
.
info
(
f
"
Skipping element
{
element
.
id
}
as it was already processed
"
)
continue
self
.
process_element
(
element
)
self
.
update_activity
(
element
.
id
,
ActivityState
.
Processed
)
except
Exception
as
e
:
# Handle errors occurring while retrieving, processing or patching the activity for this element.
# Count the element as failed in case the activity update to "started" failed with no conflict.
...
...
@@ -188,13 +179,14 @@ class ElementsWorker(
def
update_activity
(
self
,
element_id
,
state
):
"""
Update worker activity for this element
This method should not raise a runtime exception, but simply warn users
Returns False when there is a conflict initializing the activity
Otherwise return True or the response payload
"""
if
not
self
.
store_activity
:
logger
.
debug
(
"
Activity is not stored as the feature is disabled on this process
"
)
return
return
True
assert
element_id
and
isinstance
(
element_id
,
(
uuid
.
UUID
,
str
)
...
...
@@ -203,7 +195,7 @@ class ElementsWorker(
if
self
.
is_read_only
:
logger
.
warning
(
"
Cannot update activity as this worker is in read-only mode
"
)
return
return
True
try
:
out
=
self
.
request
(
...
...
@@ -216,6 +208,17 @@ class ElementsWorker(
},
)
except
ErrorResponse
as
e
:
if
state
==
ActivityState
.
Started
and
e
.
status_code
==
409
:
# 409 conflict error when updating the state of an activity to "started" mean that we
# cannot process this element. We assume that the reason is that the state transition
# was forbidden i.e. that the activity was already in a started or processed state.
# This allow concurrent access to an element activity between multiple processes.
# Element should not be counted as failed as it is probably handled somewhere else.
logger
.
debug
(
f
"
Cannot start processing element
{
element_id
}
due to a conflict.
"
f
"
Another process could have processed it with the same version already.
"
)
return
False
logger
.
warning
(
f
"
Failed to update activity of element
{
element_id
}
to
{
state
.
value
}
due to an API error:
{
e
.
content
}
"
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment