Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
Matrix
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Container Registry
Model registry
Operate
Environments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
TeDomum
Matrix
Commits
2f18a264
Commit
2f18a264
authored
7 years ago
by
Erik Johnston
Browse files
Options
Downloads
Patches
Plain Diff
Make all fields private
parent
dc519602
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
synapse/util/file_consumer.py
+31
-31
31 additions, 31 deletions
synapse/util/file_consumer.py
with
31 additions
and
31 deletions
synapse/util/file_consumer.py
+
31
−
31
View file @
2f18a264
...
...
@@ -35,24 +35,24 @@ class BackgroundFileConsumer(object):
_RESUME_ON_QUEUE_SIZE
=
2
def
__init__
(
self
,
file_obj
):
self
.
file_obj
=
file_obj
self
.
_
file_obj
=
file_obj
# Producer we're registered with
self
.
producer
=
None
self
.
_
producer
=
None
# True if PushProducer, false if PullProducer
self
.
streaming
=
False
# For PushProducers, indicates whether we've paused the producer and
# need to call resumeProducing before we get more data.
self
.
paused_producer
=
False
self
.
_
paused_producer
=
False
# Queue of slices of bytes to be written. When producer calls
# unregister a final None is sent.
self
.
bytes_queue
=
Queue
.
Queue
()
self
.
_
bytes_queue
=
Queue
.
Queue
()
# Deferred that is resolved when finished writing
self
.
finished_deferred
=
None
self
.
_
finished_deferred
=
None
# If the _writer thread throws an exception it gets stored here.
self
.
_write_exception
=
None
...
...
@@ -69,21 +69,21 @@ class BackgroundFileConsumer(object):
streaming (bool): True if push based producer, False if pull
based.
"""
if
self
.
producer
:
if
self
.
_
producer
:
raise
Exception
(
"
registerProducer called twice
"
)
self
.
producer
=
producer
self
.
_
producer
=
producer
self
.
streaming
=
streaming
self
.
finished_deferred
=
threads
.
deferToThread
(
self
.
_writer
)
self
.
_
finished_deferred
=
threads
.
deferToThread
(
self
.
_writer
)
if
not
streaming
:
self
.
producer
.
resumeProducing
()
self
.
_
producer
.
resumeProducing
()
def
unregisterProducer
(
self
):
"""
Part of IProducer interface
"""
self
.
producer
=
None
if
not
self
.
finished_deferred
.
called
:
self
.
bytes_queue
.
put_nowait
(
None
)
self
.
_
producer
=
None
if
not
self
.
_
finished_deferred
.
called
:
self
.
_
bytes_queue
.
put_nowait
(
None
)
def
write
(
self
,
bytes
):
"""
Part of IProducer interface
...
...
@@ -91,65 +91,65 @@ class BackgroundFileConsumer(object):
if
self
.
_write_exception
:
raise
self
.
_write_exception
if
self
.
finished_deferred
.
called
:
if
self
.
_
finished_deferred
.
called
:
raise
Exception
(
"
consumer has closed
"
)
self
.
bytes_queue
.
put_nowait
(
bytes
)
self
.
_
bytes_queue
.
put_nowait
(
bytes
)
# If this is a PushProducer and the queue is getting behind
# then we pause the producer.
if
self
.
streaming
and
self
.
bytes_queue
.
qsize
()
>=
self
.
_PAUSE_ON_QUEUE_SIZE
:
self
.
paused_producer
=
True
self
.
producer
.
pauseProducing
()
if
self
.
streaming
and
self
.
_
bytes_queue
.
qsize
()
>=
self
.
_PAUSE_ON_QUEUE_SIZE
:
self
.
_
paused_producer
=
True
self
.
_
producer
.
pauseProducing
()
def
_writer
(
self
):
"""
This is run in a background thread to write to the file.
"""
try
:
while
self
.
producer
or
not
self
.
bytes_queue
.
empty
():
while
self
.
_
producer
or
not
self
.
_
bytes_queue
.
empty
():
# If we've paused the producer check if we should resume the
# producer.
if
self
.
producer
and
self
.
paused_producer
:
if
self
.
bytes_queue
.
qsize
()
<=
self
.
_RESUME_ON_QUEUE_SIZE
:
if
self
.
_
producer
and
self
.
_
paused_producer
:
if
self
.
_
bytes_queue
.
qsize
()
<=
self
.
_RESUME_ON_QUEUE_SIZE
:
reactor
.
callFromThread
(
self
.
_resume_paused_producer
)
if
self
.
_notify_empty_deferred
and
self
.
bytes_queue
.
empty
():
if
self
.
_notify_empty_deferred
and
self
.
_
bytes_queue
.
empty
():
reactor
.
callFromThread
(
self
.
_notify_empty
)
bytes
=
self
.
bytes_queue
.
get
()
bytes
=
self
.
_
bytes_queue
.
get
()
# If we get a None (or empty list) then that's a signal used
# to indicate we should check if we should stop.
if
bytes
:
self
.
file_obj
.
write
(
bytes
)
self
.
_
file_obj
.
write
(
bytes
)
# If its a pull producer then we need to explicitly ask for
# more stuff.
if
not
self
.
streaming
and
self
.
producer
:
reactor
.
callFromThread
(
self
.
producer
.
resumeProducing
)
if
not
self
.
streaming
and
self
.
_
producer
:
reactor
.
callFromThread
(
self
.
_
producer
.
resumeProducing
)
except
Exception
as
e
:
self
.
_write_exception
=
e
raise
finally
:
self
.
file_obj
.
close
()
self
.
_
file_obj
.
close
()
def
wait
(
self
):
"""
Returns a deferred that resolves when finished writing to file
"""
return
make_deferred_yieldable
(
self
.
finished_deferred
)
return
make_deferred_yieldable
(
self
.
_
finished_deferred
)
def
_resume_paused_producer
(
self
):
"""
Gets called if we should resume producing after being paused
"""
if
self
.
paused_producer
and
self
.
producer
:
self
.
paused_producer
=
False
self
.
producer
.
resumeProducing
()
if
self
.
_
paused_producer
and
self
.
_
producer
:
self
.
_
paused_producer
=
False
self
.
_
producer
.
resumeProducing
()
def
_notify_empty
(
self
):
"""
Called when the _writer thread thinks the queue may be empty and
we should notify anything waiting on `wait_for_writes`
"""
if
self
.
_notify_empty_deferred
and
self
.
bytes_queue
.
empty
():
if
self
.
_notify_empty_deferred
and
self
.
_
bytes_queue
.
empty
():
d
=
self
.
_notify_empty_deferred
self
.
_notify_empty_deferred
=
None
d
.
callback
(
None
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment