Codebase list ruby-parallel / f130335
Update upstream source from tag 'upstream/1.20.1' Update to upstream version '1.20.1' with Debian dir fe456cc95fe6c0b8576ba49d4f373b51dc1f1f29 Hideki Yamane 3 years ago
15 changed file(s) with 162 addition(s) and 147 deletion(s). Raw diff Collapse all Expand all
+23
-0
.github/workflows/actions.yml less more
0 name: CI
1 on:
2 push:
3 branches: [master]
4 pull_request:
5 branches: [master]
6 jobs:
7 test:
8 runs-on: ubuntu-latest
9 services:
10 mysql:
11 image: mysql
12 strategy:
13 matrix:
14 ruby: [ '2.5', '2.6', '2.7' ]
15 name: ${{ matrix.ruby }} rake ${{ matrix.task }}
16 steps:
17 - uses: actions/checkout@v2
18 - uses: ruby/setup-ruby@v1
19 with:
20 ruby-version: ${{ matrix.ruby }}
21 bundler-cache: true # runs 'bundle install' and caches installed gems automatically
22 - run: bundle exec rake
+0
-19
.travis.yml less more
0 language: ruby
1 dist: precise
2 branches:
3 only: master
4 rvm:
5 - 2.2
6 - 2.3
7 - 2.4
8 - 2.5
9 - 2.6
10
11 matrix:
12 include:
13 - rvm: 2.6
14 env: RUBYOPT="--jit"
15 allow_failures:
16 - env: RUBYOPT="--jit"
17
18 before_install: ruby -e "File.write('Gemfile.lock', File.read('Gemfile.lock').split('BUNDLED WITH').first)"
+2
-2
Gemfile less more
33 gem 'bump'
44 gem 'rake'
55 gem 'rspec'
6 gem 'activerecord', "~>4.2.8"
6 gem 'activerecord', "~> 6.0"
77 gem 'ruby-progressbar'
88 gem 'rspec-rerun'
99 gem 'rspec-legacy_formatters'
1010
11 gem 'mysql2', :group => :mysql
11 gem 'mysql2', group: :mysql
1212 gem 'sqlite3'
+21
-21
Gemfile.lock less more
00 PATH
11 remote: .
22 specs:
3 parallel (1.19.1)
3 parallel (1.20.1)
44
55 GEM
66 remote: https://rubygems.org/
77 specs:
8 activemodel (4.2.8)
9 activesupport (= 4.2.8)
10 builder (~> 3.1)
11 activerecord (4.2.8)
12 activemodel (= 4.2.8)
13 activesupport (= 4.2.8)
14 arel (~> 6.0)
15 activesupport (4.2.8)
16 i18n (~> 0.7)
8 activemodel (6.0.3.1)
9 activesupport (= 6.0.3.1)
10 activerecord (6.0.3.1)
11 activemodel (= 6.0.3.1)
12 activesupport (= 6.0.3.1)
13 activesupport (6.0.3.1)
14 concurrent-ruby (~> 1.0, >= 1.0.2)
15 i18n (>= 0.7, < 2)
1716 minitest (~> 5.1)
18 thread_safe (~> 0.3, >= 0.3.4)
1917 tzinfo (~> 1.1)
20 arel (6.0.4)
21 builder (3.2.3)
18 zeitwerk (~> 2.2, >= 2.2.2)
2219 bump (0.5.3)
20 concurrent-ruby (1.1.6)
2321 diff-lcs (1.3)
24 i18n (0.8.1)
25 minitest (5.10.1)
26 mysql2 (0.5.2)
27 rake (12.0.0)
22 i18n (1.8.3)
23 concurrent-ruby (~> 1.0)
24 minitest (5.14.1)
25 mysql2 (0.5.3)
26 rake (13.0.1)
2827 rspec (3.6.0)
2928 rspec-core (~> 3.6.0)
3029 rspec-expectations (~> 3.6.0)
4342 rspec (~> 3.0)
4443 rspec-support (3.6.0)
4544 ruby-progressbar (1.8.1)
46 sqlite3 (1.3.13)
45 sqlite3 (1.4.2)
4746 thread_safe (0.3.6)
4847 thread_safe (0.3.6-java)
49 tzinfo (1.2.3)
48 tzinfo (1.2.7)
5049 thread_safe (~> 0.1)
50 zeitwerk (2.3.0)
5151
5252 PLATFORMS
5353 java
5454 ruby
5555
5656 DEPENDENCIES
57 activerecord (~> 4.2.8)
57 activerecord (~> 6.0)
5858 bump
5959 mysql2
6060 parallel!
6666 sqlite3
6767
6868 BUNDLED WITH
69 2.0.1
69 2.1.4
+17
-4
Readme.md less more
4242 Parallel.each( -> { items.pop || Parallel::Stop }) { |number| ... }
4343 ```
4444
45 You can also call `any?` or `all?`, which work the same way as `Array#any?` and `Array#all?`.
45 Also supports `any?` or `all?`
4646
4747 ```Ruby
4848 Parallel.any?([1,2,3,4,5,6,7]) { |number| number == 4 }
5151 Parallel.all?([1,2,nil,4,5]) { |number| number != nil }
5252 # => false
5353 ```
54
5554
5655 Processes/Threads are workers, they grab the next piece of work when they finish.
5756
6968
7069 ### ActiveRecord
7170
72 Try any of those to get working parallel AR
71 #### Connection Lost
72
73 - Multithreading needs connection pooling, forks need reconnects
74 - Adjust connection pool size in `config/database.yml` when multithreading
7375
7476 ```Ruby
7577 # reproducibly fixes things (spec/cases/map_with_ar.rb)
9294 end
9395 ```
9496
97 #### NameError: uninitialized constant
98
99 A race happens when ActiveRecord models are autoloaded inside parallel threads
100 in environments that lazy-load, like development, test, or migrations.
101
102 To fix, autoloaded classes before the parallel block with either `require '<modelname>'` or `ModelName.class`.
103
95104 ### Break
96105
97106 ```Ruby
98 Parallel.map(User.all) do |user|
107 Parallel.map([1, 2, 3]) do |i|
99108 raise Parallel::Break # -> stops after all current items are finished
100109 end
110 ```
111
112 ```Ruby
113 Parallel.map([1, 2, 3]) { |i| raise Parallel::Break, i if i == 2 } == 2
101114 ```
102115
103116 ### Kill
+0
-21
gem-public_cert.pem less more
0 -----BEGIN CERTIFICATE-----
1 MIIDcDCCAligAwIBAgIBATANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo
2 YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0
3 MB4XDTE0MDIwNDIwMjk0MVoXDTE1MDIwNDIwMjk0MVowPzEQMA4GA1UEAwwHbWlj
4 aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp
5 dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H
6 MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO
7 cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO
8 6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh
9 h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX
10 FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R
11 /88CAwEAAaN3MHUwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0OBBYEFLIj
12 Z1x7SnjGGHK+MiVZkFjjS/iMMB0GA1UdEQQWMBSBEm1pY2hhZWxAZ3Jvc3Nlci5p
13 dDAdBgNVHRIEFjAUgRJtaWNoYWVsQGdyb3NzZXIuaXQwDQYJKoZIhvcNAQEFBQAD
14 ggEBAExBcUWfGuamYn+IddOA0Ws8jUKwB14RXoZRDrTiTAlMm3Bkg2OKyxS3uJXa
15 6Z+LwFiZwVYk62yHXqNzEJycQk4SEmY+xDWLj0p7X6qEeU4QZKwR1TwJ5z3PTrZ6
16 irJgM3q7NIBRvmTzRaAghWcQn+Eyr5YLOfMksjVBMUMnzh5/ZDgq53LphgJbGwvz
17 ScJAgfNclLHnjk9q1mT1s0e1FPWbiAL3siBIR5HpH8qtSEiivTf2ntciebOqS93f
18 F5etKHZg0j3eHO31/i2HnswY04lqGImUu6aM5EnijFTB7PPW2KwKKM4+kKDYFdlw
19 /0WV1Ng2/Y6qsHwmqGg2VlYj2h4=
20 -----END CERTIFICATE-----
+1
-1
lib/parallel/version.rb less more
00 module Parallel
1 VERSION = Version = '1.19.1'
1 VERSION = Version = '1.20.1'
22 end
+35
-35
lib/parallel.rb less more
22 require 'parallel/processor_count'
33
44 module Parallel
5 extend Parallel::ProcessorCount
5 extend ProcessorCount
6
7 Stop = Object.new.freeze
68
79 class DeadWorker < StandardError
810 end
911
1012 class Break < StandardError
11 end
12
13 class Kill < StandardError
13 attr_reader :value
14 def initialize(value = nil)
15 @value = value
16 end
17 end
18
19 class Kill < Break
1420 end
1521
1622 class UndumpableException < StandardError
2026 @backtrace = original.backtrace
2127 end
2228 end
23
24 Stop = Object.new.freeze
2529
2630 class ExceptionWrapper
2731 attr_reader :exception
101105 item, index = @mutex.synchronize do
102106 return if @stopped
103107 item = @lambda.call
104 @stopped = (item == Parallel::Stop)
108 @stopped = (item == Stop)
105109 return if @stopped
106110 [item, @index += 1]
107111 end
200204
201205 class << self
202206 def in_threads(options={:count => 2})
207 threads = []
208 count, _ = extract_count_from_options(options)
209
203210 Thread.handle_interrupt(Exception => :never) do
204211 begin
205 threads = []
206 count, _ = extract_count_from_options(options)
207 count.times do |i|
208 threads << Thread.new { yield(i) }
209 end
210212 Thread.handle_interrupt(Exception => :immediate) do
213 count.times do |i|
214 threads << Thread.new { yield(i) }
215 end
211216 threads.map(&:value)
212217 end
213218 ensure
228233
229234 def any?(*args, &block)
230235 raise "You must provide a block when calling #any?" if block.nil?
231 !each(*args) { |*a| raise Parallel::Kill if block.call(*a) }
236 !each(*args) { |*a| raise Kill if block.call(*a) }
232237 end
233238
234239 def all?(*args, &block)
235240 raise "You must provide a block when calling #all?" if block.nil?
236 !!each(*args) { |*a| raise Parallel::Kill unless block.call(*a) }
241 !!each(*args) { |*a| raise Kill unless block.call(*a) }
237242 end
238243
239244 def each_with_index(array, options={}, &block)
268273 options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
269274 add_progress_bar!(job_factory, options)
270275
271 results = if size == 0
272 work_direct(job_factory, options, &block)
273 elsif method == :in_threads
274 work_in_threads(job_factory, options.merge(:count => size), &block)
275 else
276 work_in_processes(job_factory, options.merge(:count => size), &block)
277 end
278 if results
279 options[:return_results] ? results : source
280 end
276 result =
277 if size == 0
278 work_direct(job_factory, options, &block)
279 elsif method == :in_threads
280 work_in_threads(job_factory, options.merge(:count => size), &block)
281 else
282 work_in_processes(job_factory, options.merge(:count => size), &block)
283 end
284
285 return result.value if result.is_a?(Break)
286 raise result if result.is_a?(Exception)
287 options[:return_results] ? result : source
281288 end
282289
283290 def map_with_index(array, options={}, &block)
338345 rescue
339346 exception = $!
340347 end
341 handle_exception(exception, results)
348 exception || results
342349 ensure
343350 self.worker_number = nil
344351 end
365372 end
366373 end
367374
368 handle_exception(exception, results)
375 exception || results
369376 end
370377
371378 def work_in_processes(job_factory, options, &blk)
399406 results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
400407 rescue StandardError => e
401408 exception = e
402 if Parallel::Kill === exception
409 if Kill === exception
403410 (workers - [worker]).each do |w|
404411 w.thread.kill if w.thread
405412 UserInterruptHandler.kill(w.pid)
412419 end
413420 end
414421 end
415
416 handle_exception(exception, results)
422 exception || results
417423 end
418424
419425 def replace_worker(job_factory, workers, i, options, blk)
482488 end
483489 end
484490
485 def handle_exception(exception, results)
486 return nil if [Parallel::Break, Parallel::Kill].include? exception.class
487 raise exception if exception
488 results
489 end
490
491491 # options is either a Integer or a Hash with :count
492492 def extract_count_from_options(options)
493493 if options.is_a?(Hash)
+1
-1
parallel.gemspec less more
1414 }
1515 s.files = `git ls-files lib MIT-LICENSE.txt`.split("\n")
1616 s.license = "MIT"
17 s.required_ruby_version = '>= 2.2'
17 s.required_ruby_version = '>= 2.4'
1818 end
+4
-5
spec/cases/count_open_pipes.rb less more
00 require './spec/cases/helper'
1
2 results = Parallel.map(Array.new(20), :in_processes => 20) do
3 `lsof | grep pipe | wc -l`.to_i
4 end
5 puts results.max
1 count = ->(*) { `lsof | grep pipe | wc -l`.to_i }
2 start = count.()
3 results = Parallel.map(Array.new(20), :in_processes => 20, &count)
4 puts results.max - start
+2
-2
spec/cases/each_with_ar_sqlite.rb less more
00 require './spec/cases/helper'
11 require "active_record"
22 require "sqlite3"
3 require "tempfile"
34 STDOUT.sync = true
45 in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
56
6 Tempfile.open("db") do |temp|
7
7 Tempfile.create("db") do |temp|
88 ActiveRecord::Schema.verbose = false
99 ActiveRecord::Base.establish_connection(
1010 :adapter => "sqlite3",
+27
-29
spec/cases/map_with_ar.rb less more
00 require './spec/cases/helper'
11 require "active_record"
22
3 Tempfile.open("xxx") do |f|
4 database = "parallel_with_ar_test"
5 `mysql #{database} -e '' || mysql -e 'create database #{database}'`
3 database = "parallel_with_ar_test"
4 `mysql #{database} -e '' || mysql -e 'create database #{database}'`
65
7 ActiveRecord::Schema.verbose = false
8 ActiveRecord::Base.establish_connection(
9 :adapter => "mysql2",
10 :database => database
11 )
6 ActiveRecord::Schema.verbose = false
7 ActiveRecord::Base.establish_connection(
8 :adapter => "mysql2",
9 :database => database
10 )
1211
13 class User < ActiveRecord::Base
14 end
12 class User < ActiveRecord::Base
13 end
1514
16 # create tables
17 unless User.table_exists?
18 ActiveRecord::Schema.define(:version => 1) do
19 create_table :users do |t|
20 t.string :name
21 end
15 # create tables
16 unless User.table_exists?
17 ActiveRecord::Schema.define(:version => 1) do
18 create_table :users do |t|
19 t.string :name
2220 end
2321 end
22 end
2423
25 User.delete_all
24 User.delete_all
2625
27 User.create!(:name => "X")
26 User.create!(:name => "X")
2827
29 Parallel.map(1..8) do |i|
30 User.create!(:name => i)
31 end
28 Parallel.map(1..8) do |i|
29 User.create!(:name => i)
30 end
3231
33 puts "User.count: #{User.count}"
32 puts "User.count: #{User.count}"
3433
35 puts User.connection.reconnect!.inspect
34 puts User.connection.reconnect!.inspect
3635
37 Parallel.map(1..8, :in_threads => 4) do |i|
38 User.create!(:name => i)
39 end
36 Parallel.map(1..8, :in_threads => 4) do |i|
37 User.create!(:name => i)
38 end
4039
41 User.create!(:name => "X")
40 User.create!(:name => "X")
4241
43 puts User.all.map(&:name).sort.join("-")
44 end
42 puts User.all.map(&:name).sort.join("-")
+12
-0
spec/cases/timeout_in_threads.rb less more
0 require './spec/cases/helper'
1 require 'timeout'
2
3 Parallel.each([1], in_threads: 1) do |i|
4 begin
5 Timeout.timeout(0.1) { sleep 0.2 }
6 rescue Timeout::Error
7 puts "OK"
8 else
9 puts "BROKEN"
10 end
11 end
+1
-1
spec/cases/with_break.rb less more
77 result = Parallel.public_send(method, 1..100, in_worker_type => worker_size) do |x|
88 sleep 0.1 # so all workers get started
99 print x
10 raise Parallel::Break if x == 1
10 raise Parallel::Break, *ARGV if x == 1
1111 sleep 0.2 # so now no work gets queued before Parallel::Break is raised
1212 x
1313 end
+16
-6
spec/parallel_spec.rb less more
178178 end
179179
180180 it 'can handle to high fork rate' do
181 unless RbConfig::CONFIG["target_os"] =~ /darwin1/
182 `ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
183 end
181 next if RbConfig::CONFIG["target_os"].include?("darwin1") # kills macs for some reason
182 `ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
184183 end
185184
186185 it 'does not leave processes behind while running' do
189188 end
190189
191190 it "does not open unnecessary pipes" do
192 open_pipes = `lsof | grep pipe | wc -l`.to_i
193 max_pipes = `ruby spec/cases/count_open_pipes.rb`.to_i
194 (max_pipes - open_pipes).should < 400
191 max = (RbConfig::CONFIG["target_os"].include?("darwin1") ? 10 : 1800) # somehow super bad on CI
192 `ruby spec/cases/count_open_pipes.rb`.to_i.should < max
195193 end
196194 end
197195
211209 end
212210
213211 it "raises when a thread raises" do
212 Thread.report_on_exception = false
214213 lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
214 ensure
215 Thread.report_on_exception = true
215216 end
216217 end
217218
283284
284285 it "does not call the finish hook when a start hook fails with #{type}" do
285286 `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1`.should == '3 called'
287 end
288
289 it "can return from break with #{type}" do
290 `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break.rb hi 2>&1`.should =~ /^\d{4} Parallel::Break raised - result "hi"$/
286291 end
287292
288293 it "sets Parallel.worker_number with 4 #{type}" do
466471 out = `ruby spec/cases/map_worker_number_isolation.rb`
467472 out.should == "0,1\nOK"
468473 end
474
475 it 'can use Timeout' do
476 out = `ruby spec/cases/timeout_in_threads.rb`
477 out.should == "OK\n"
478 end
469479 end
470480
471481 describe ".map_with_index" do