Codebase list libjavaewah-java / 0cf2a2a
Import upstream version 1.1.7+git20210928.1.cbebdfb Debian Janitor 1 year, 6 months ago
14 changed file(s) with 889 addition(s) and 107 deletion(s). Raw diff Collapse all Expand all
0 name: Java CI
1
2 on: [push, pull_request]
3
4 jobs:
5 build:
6 runs-on: ubuntu-latest
7
8 steps:
9 - uses: actions/checkout@v2
10 - name: Set up JDK 16
11 uses: actions/setup-java@v2
12 with:
13 java-version: '16'
14 distribution: 'adopt'
15 - name: Build with Maven
16 run: mvn test
1212
1313 script: mvn clean test
1414
15
16 cache:
17 directories:
18 - $HOME/.m2
00 JavaEWAH
11 ==========================================================
2 [![Build Status](https://travis-ci.org/lemire/javaewah.png)](https://travis-ci.org/lemire/javaewah)
2 [![Java CI](https://github.com/lemire/javaewah/actions/workflows/basic.yml/badge.svg)](https://github.com/lemire/javaewah/actions/workflows/basic.yml)
33 [![][maven img]][maven]
44 [![][license img]][license]
55 [![docs-badge][]][docs]
66 [![Coverage Status](https://coveralls.io/repos/lemire/javaewah/badge.svg?branch=master)](https://coveralls.io/r/lemire/javaewah?branch=master)
77 [![Code Quality: Cpp](https://img.shields.io/lgtm/grade/java/g/lemire/javaewah.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/lemire/javaewah/context:java)
88
9 (c) 2009-2016
10 Daniel Lemire (http://lemire.me/en/),
11 Cliff Moon,
12 David McIntosh (https://github.com/mctofu),
13 Robert Becho (https://github.com/RBecho),
14 Colby Ranger (https://github.com/crangeratgoogle),
15 Veronika Zenz (https://github.com/veronikazenz),
16 Owen Kaser (https://github.com/owenkaser),
17 Gregory Ssi-Yan-Kai (https://github.com/gssiyankai),
18 and Rory Graves (https://github.com/rorygraves)
19
20
21 This code is licensed under Apache License, Version 2.0 (ASL2.0).
22 (GPL 2.0 derivatives are allowed.)
239
2410 This is a word-aligned compressed variant of
2511 the Java Bitset class. We provide both a 64-bit
2612 and a 32-bit RLE-like compression scheme. It can
27 be used to implement bitmap indexes.
13 be used to implement bitmap indexes. The EWAH format
14 it relies upon is used in the git implementation
15 that runs GitHub.
2816
2917 The goal of word-aligned compression is not to
3018 achieve the best compression, but rather to
9583 magnitude faster than other possible implementation of a set (e.g., as a hash set)
9684 while using several times less memory.
9785
98 However, a bitset, even a compressed one is not always applicable. For example, if the
86 However, a bitset, even a compressed one is not always applicable. For example, if
9987 you have 1000 random-looking integers, then a simple array might be the best representation.
10088 We refer to this case as the "sparse" scenario.
10189
152140 For more details regarding the compression format, please
153141 see Section 3 of the following paper:
154142
155 Daniel Lemire, Owen Kaser, Kamel Aouiche, Sorting improves word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages 3-28, 2010.
156 http://arxiv.org/abs/0901.3751
157
158
159
160 (The PDF file is freely available on the arXiv site.)
143 Daniel Lemire, Owen Kaser, Kamel Aouiche, [Sorting improves word-aligned bitmap indexes](http://arxiv.org/abs/0901.3751). Data & Knowledge Engineering 69 (1), pages 3-28, 2010.
144
161145
162146 Benchmark
163147 ---------
172156 Unit testing
173157 ------------
174158
175 As of October 2011, this packages relies on Maven. To
159 As of October 2011, this package relies on Maven. To
176160 test it:
177161
162 ```
178163 mvn test
164 ```
179165
180166 See
181167 http://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
182168 for details.
169
170 We support Java 8 and up, but to build the library, you need Java 9 and up
171 with the default Maven setup, since we rely on the `--release` flag functionality
172 (unavailable in Java 8) to sidestep the [Deaded-NoSuchMethodError issue](https://www.morling.dev/blog/bytebuffer-and-the-dreaded-nosuchmethoderror/).
173
183174
184175
185176 Usage
288279
289280 sudo apt-get install libjavaewah-java
290281
291 Travis (Continuous integration)
292 -------------------------------
293
294 You can check whether the latest version builds on your favorite version
295 of Java using Travis: https://travis-ci.org/lemire/javaewah/builds/
296
297282 Clojure
298283 -------
299284
342327 Further reading
343328 ---------------
344329
345 Daniel Lemire, Owen Kaser, Kamel Aouiche, Sorting improves word-aligned bitmap indexes, Data & Knowledge Engineering 69 (1), 2010.
346 http://arxiv.org/abs/0901.3751
347
348 Owen Kaser and Daniel Lemire, Compressed bitmap indexes: beyond unions and intersections, Software: Practice and Experience 46 (2), 2016.
349 http://arxiv.org/abs/1402.4466
350
351
330 - Daniel Lemire, Owen Kaser, Kamel Aouiche, [Sorting improves word-aligned bitmap indexes](http://arxiv.org/abs/0901.3751), Data & Knowledge Engineering 69 (1), 2010.
331 - Owen Kaser and Daniel Lemire, [Compressed bitmap indexes: beyond unions and intersections](http://arxiv.org/abs/1402.4466), Software: Practice and Experience 46 (2), 2016.
332
333
334 Credit
335 --------
336
337
338 (c) 2009-2021
339 [Daniel Lemire](http://lemire.me/en/), Cliff Moon, [David McIntosh](https://github.com/mctofu), [Robert Becho](https://github.com/RBecho), [Colby Ranger](https://github.com/crangeratgoogle), [Veronika Zenz](https://github.com/veronikazenz), [Owen Kaser](https://github.com/owenkaser), [Gregory Ssi-Yan-Kai](https://github.com/gssiyankai), and [Rory Graves](https://github.com/rorygraves)
340
341
342 This code is licensed under Apache License, Version 2.0 (ASL2.0).
343 (GPL 2.0 derivatives are allowed.)
352344
353345 Acknowledgement
354346 ---------------
11 <modelVersion>4.0.0</modelVersion>
22 <groupId>com.googlecode.javaewah</groupId>
33 <artifactId>JavaEWAH</artifactId>
4 <version>1.1.7</version>
4 <version>1.1.14-SNAPSHOT</version>
55 <packaging>bundle</packaging>
66 <properties>
7 <maven.compiler.source>1.8</maven.compiler.source>
8 <maven.compiler.target>1.8</maven.compiler.target>
7 <maven.compiler.release>8</maven.compiler.release>
98 <encoding>UTF-8</encoding>
109 </properties>
1110 <licenses>
4443 <dependency>
4544 <groupId>junit</groupId>
4645 <artifactId>junit</artifactId>
47 <version>4.10</version>
46 <version>4.13.1</version>
4847 <scope>test</scope>
4948 </dependency>
5049 </dependencies>
6463 <artifactId>maven-surefire-plugin</artifactId>
6564 <version>2.19.1</version>
6665 <configuration>
66 <enableAssertions>true</enableAssertions>
67 <trimStackTrace>false</trimStackTrace>
6768 <forkCount>3</forkCount>
6869 <reuseForks>true</reuseForks>
6970 <argLine>-Xmx1024m</argLine>
512512 prey.discardFirstWords(predator.getRunningLength());
513513 } else if (i_is_prey) {
514514 final long index = prey.discharge(container, predator.getRunningLength());
515 // Todo: this may cause fragmentation whereas 0 literal words are inserted.
515516 container.addStreamOfEmptyWords(false, predator.getRunningLength() - index);
516517 } else {
517518 final long index = prey.dischargeNegated(container, predator.getRunningLength());
11731174 }
11741175 nword += (int) rl;
11751176 long lw = RunningLengthWord.getNumberOfLiteralWords(this.buffer, pos);
1176 if(lw > 0) {
1177 long word = this.buffer.getWord(pos + 1);
1178 if(word != 0l) {
1179 long T = word & -word;
1180 return nword * WORD_IN_BITS + Long.bitCount(T - 1);
1181 }
1177 for(int p = pos + 1 ; p <= pos + lw; p++) {
1178 long word = this.buffer.getWord(p);
1179 // In theory, words should never be zero. Unfortunately due to the
1180 // design which requires us to support 'andnot' and effective universe
1181 // sizes, we sometimes end up appending zero words.
1182 if(word != 0l) {
1183 long T = word & -word;
1184 return nword * WORD_IN_BITS + Long.bitCount(T - 1);
1185 }
1186 nword++;
11821187 }
1183 }
1188 }
11841189 return -1;
11851190 }
11861191
12561261 this.sizeInBits = i + 1;
12571262 if (value) {
12581263 if (dist > 0) {
1264 // Let us trim the lone zero word if needed
1265 if (this.rlw.getNumberOfLiteralWords() > 0 && this.buffer.getLastWord() == 0l) {
1266 this.buffer.removeLastWord();
1267 this.rlw.setNumberOfLiteralWords(this.rlw.getNumberOfLiteralWords() - 1);
1268 insertEmptyWord(false);
1269 }
12591270 if (dist > 1) {
12601271 fastaddStreamOfEmptyWords(false, dist - 1);
12611272 }
16081619
16091620 /**
16101621 * A more detailed string describing the bitmap (useful for debugging).
1622 * A JSON output is produced.
16111623 *
16121624 * @return the string
16131625 */
16141626 public String toDebugString() {
16151627 StringBuilder ans = new StringBuilder();
1616 ans.append(" EWAHCompressedBitmap, size in bits = ");
1617 ans.append(this.sizeInBits).append(" size in words = ");
1618 ans.append(this.buffer.sizeInWords()).append("\n");
1628 ans.append("{\"size in bits\":");
1629 ans.append(this.sizeInBits).append(", \"size in words\":");
1630 ans.append(this.buffer.sizeInWords()).append(",");
16191631 final EWAHIterator i = this.getEWAHIterator();
1632 ans.append(" \"content\": [");
1633 boolean first = true;
16201634 while (i.hasNext()) {
16211635 RunningLengthWord localrlw = i.next();
1636 if(!first) { ans.append(","); }
1637 first = false;
1638 ans.append("[");
1639
16221640 if (localrlw.getRunningBit()) {
1623 ans.append(localrlw.getRunningLength()).append(" 1x11\n");
1641 ans.append(localrlw.getRunningLength()).append(",").append(" \"1x11\", ");
16241642 } else {
1625 ans.append(localrlw.getRunningLength()).append(" 0x00\n");
1626 }
1627 ans.append(localrlw.getNumberOfLiteralWords()).append(" dirties\n");
1628 for (int j = 0; j < localrlw.getNumberOfLiteralWords(); ++j) {
1643 ans.append(localrlw.getRunningLength()).append(",").append(" \"0x00\", ");
1644 }
1645 ans.append("[");
1646 int j = 0;
1647 for (; j + 1 < localrlw.getNumberOfLiteralWords(); ++j) {
16291648 long data = i.buffer().getWord(i.literalWords() + j);
1630 ans.append("\t").append(data).append("\n");
1631 }
1632 }
1649 ans.append("\"0x").append(Long.toHexString(data)).append("\",");
1650 }
1651 if(j < localrlw.getNumberOfLiteralWords()) {
1652 long data = i.buffer().getWord(i.literalWords() + j);
1653 ans.append("\"0x").append(Long.toHexString(data)).append("\"");
1654 }
1655 ans.append("]]");
1656 }
1657 ans.append("]}");
16331658 return ans.toString();
16341659 }
16351660
20692094 int fullwords = b / WORD_IN_BITS;
20702095 int shift = b % WORD_IN_BITS;
20712096 answer.addStreamOfEmptyWords(false, fullwords);
2072 if (shift == 0) {
2073 answer.buffer.push_back(this.buffer, 0, sz);
2097 if (shift == 0) {
2098 while (true) {
2099 long rl = i.getRunningLength();
2100 if (rl > 0) {
2101 answer.addStreamOfEmptyWords(i.getRunningBit(), rl);
2102 }
2103 int x = i.getNumberOfLiteralWords();
2104 for (int k = 0; k < x; ++k) {
2105 answer.addWord(i.getLiteralWordAt(k));
2106 }
2107 if (!i.next()) {
2108 break;
2109 }
2110 }
20742111 } else {
20752112 // whether the shift should justify a new word
20762113 final boolean shiftextension = ((this.sizeInBits + WORD_IN_BITS - 1) % WORD_IN_BITS) + shift >= WORD_IN_BITS;
507507 prey.discardFirstWords(predator
508508 .getRunningLength());
509509 } else if (i_is_prey) {
510 // Todo: this may cause fragmentation whereas 0 literal words are inserted.
510511 final int index = prey.discharge(container,
511512 predator.getRunningLength());
512513 container.addStreamOfEmptyWords(false,
11771178 }
11781179 nword += rl;
11791180 int lw = RunningLengthWord32.getNumberOfLiteralWords(this.buffer, pos);
1180 if(lw > 0) {
1181 int word = this.buffer.getWord(pos + 1);
1182 if(word != 0) {
1183 int T = word & -word;
1184 return nword * WORD_IN_BITS + Integer.bitCount(T - 1);
1185 }
1186 }
1181 for(int p = pos + 1 ; p <= pos + lw; p++) {
1182 int word = this.buffer.getWord(p);
1183 // In theory, words should never be zero. Unfortunately due to the
1184 // design which requires us to support 'andnot' and effective universe
1185 // sizes, we sometimes end up appending zero words.
1186 if(word != 0l) {
1187 int T = word & -word;
1188 return nword * WORD_IN_BITS + Integer.bitCount(T - 1);
1189 }
1190 nword++;
1191 }
11871192 }
11881193 return -1;
11891194 }
12611266 this.sizeInBits = i + 1;
12621267 if(value) {
12631268 if (dist > 0) {
1269 // Let us trim the lone zero word if needed
1270 if (this.rlw.getNumberOfLiteralWords() > 0 && this.buffer.getLastWord() == 0) {
1271 this.buffer.removeLastWord();
1272 this.rlw.setNumberOfLiteralWords(this.rlw.getNumberOfLiteralWords() - 1);
1273 insertEmptyWord(false);
1274 }
12641275 if (dist > 1) {
12651276 fastaddStreamOfEmptyWords(false, dist - 1);
12661277 }
16201631
16211632 /**
16221633 * A more detailed string describing the bitmap (useful for debugging).
1634 * A JSON output is produced.
16231635 *
16241636 * @return the string
16251637 */
1638
16261639 public String toDebugString() {
1627 StringBuffer sb = new StringBuffer(" EWAHCompressedBitmap, size in bits = ");
1628 sb.append(this.sizeInBits).append(" size in words = ");
1629 sb.append(this.buffer.sizeInWords()).append("\n");
1640 StringBuilder ans = new StringBuilder();
1641 ans.append("{\"size in bits\":");
1642 ans.append(this.sizeInBits).append(", \"size in words\":");
1643 ans.append(this.buffer.sizeInWords()).append(",");
16301644 final EWAHIterator32 i = this.getEWAHIterator();
1645 ans.append(" \"content\": [");
1646 boolean first = true;
16311647 while (i.hasNext()) {
16321648 RunningLengthWord32 localrlw = i.next();
1649 if(!first) { ans.append(","); }
1650 first = false;
1651 ans.append("[");
1652
16331653 if (localrlw.getRunningBit()) {
1634 sb.append(localrlw.getRunningLength()).append(" 1x11\n");
1654 ans.append(localrlw.getRunningLength()).append(",").append(" \"1x11\", ");
16351655 } else {
1636 sb.append(localrlw.getRunningLength()).append(" 0x00\n");
1637 }
1638 sb.append(localrlw.getNumberOfLiteralWords()).append(" dirties\n");
1639 for (int j = 0; j < localrlw.getNumberOfLiteralWords(); ++j) {
1656 ans.append(localrlw.getRunningLength()).append(",").append(" \"0x00\", ");
1657 }
1658 ans.append("[");
1659 int j = 0;
1660 for (; j + 1 < localrlw.getNumberOfLiteralWords(); ++j) {
16401661 int data = i.buffer().getWord(i.literalWords() + j);
1641 sb.append("\t").append(data).append("\n");
1642 }
1643 }
1644 return sb.toString();
1662 ans.append("\"0x").append(Integer.toHexString(data)).append("\",");
1663 }
1664 if(j < localrlw.getNumberOfLiteralWords()) {
1665 int data = i.buffer().getWord(i.literalWords() + j);
1666 ans.append("\"0x").append(Integer.toHexString(data)).append("\"");
1667 }
1668 ans.append("]]");
1669 }
1670 ans.append("]}");
1671 return ans.toString();
16451672 }
16461673
16471674 /**
20752102 int shift = b % WORD_IN_BITS;
20762103 answer.addStreamOfEmptyWords(false, fullwords);
20772104 if (shift == 0) {
2078 answer.buffer.push_back(this.buffer, 0, sz);
2105 while (true) {
2106 int rl = i.getRunningLength();
2107 if (rl > 0) {
2108 answer.addStreamOfEmptyWords(i.getRunningBit(), rl);
2109 }
2110 int x = i.getNumberOfLiteralWords();
2111 for (int k = 0; k < x; ++k) {
2112 answer.addWord(i.getLiteralWordAt(k));
2113 }
2114 if (!i.next()) {
2115 break;
2116 }
2117 }
20792118 } else {
20802119 int w = 0;
20812120 while (true) {
0 package com.googlecode.javaewah;
1
2 import java.util.BitSet;
3 import java.util.Iterator;
4 // credit @svanmald
5 public class BackwardBitSetIterator implements Iterator<Integer> {
6
7 private final BitSet bitSet;
8 private int next;
9
10 public BackwardBitSetIterator(BitSet bitSet) {
11 this.bitSet = bitSet;
12 this.next = bitSet.previousSetBit(bitSet.length());
13 }
14
15 @Override
16 public boolean hasNext() {
17 return next != -1;
18 }
19
20 @Override
21 public Integer next() {
22 int current = next;
23 next = bitSet.previousSetBit(current - 1);
24 return current;
25 }
26 }
0 package com.googlecode.javaewah;
1
2 import com.googlecode.javaewah.EWAHCompressedBitmap;
3 import com.googlecode.javaewah.IntIterator;
4 import java.util.BitSet;
5 // credit @svanmald
6 public class EWAHBitSetPair {
7
8 private EWAHCompressedBitmap bitmap;
9 private BitSet bitSet;
10
11 public EWAHBitSetPair() {
12 bitmap = new EWAHCompressedBitmap();
13 bitSet = new BitSet();
14 }
15
16 public void validate() {
17 assert bitmap.cardinality() == bitSet.cardinality();
18 ForwardBitSetIterator forwardBitSetIterator = new ForwardBitSetIterator(bitSet);
19 for (Integer current : bitmap) {
20 Integer next = forwardBitSetIterator.next();
21 assert bitmap.get(current);
22 assert next.equals(current);
23 }
24
25 BackwardBitSetIterator backwardBitSetIterator = new BackwardBitSetIterator(bitSet);
26 IntIterator reverseIterator = bitmap.reverseIntIterator();
27 while (reverseIterator.hasNext()) {
28 int nextBitMap = reverseIterator.next();
29 Integer nextBitSet = backwardBitSetIterator.next();
30 assert nextBitSet == nextBitMap;
31 }
32 assert !backwardBitSetIterator.hasNext();
33
34 EWAHCompressedBitmap result = new EWAHCompressedBitmap().or(bitmap);
35 assert result.equals(bitmap);
36 assert bitmap.equals(result);
37 assert bitmap.isEmpty() || bitmap.getFirstSetBit() == bitmap.iterator().next();
38 }
39
40 public void or(EWAHBitSetPair other) {
41 bitmap = bitmap.or(other.bitmap);
42 bitSet.or(other.bitSet);
43 }
44
45 public void and(EWAHBitSetPair other) {
46 bitmap = bitmap.and(other.bitmap);
47 bitSet.and(other.bitSet);
48 }
49
50 public void andNot(EWAHBitSetPair other) {
51 bitmap = bitmap.andNot(other.bitmap);
52 bitSet.andNot(other.bitSet);
53 }
54
55 public void xor(EWAHBitSetPair other) {
56 bitmap = bitmap.xor(other.bitmap);
57 bitSet.xor(other.bitSet);
58 }
59
60 public void set(int value) {
61 bitSet.set(value);
62 bitmap.set(value);
63 }
64
65 public void clear(int value) {
66 bitSet.clear(value);
67 bitmap.clear(value);
68 }
69 }
2020 */
2121 @SuppressWarnings("javadoc")
2222 public class EWAHCompressedBitmapTest {
23
23 @Test
24 public void issue77() {
25 EWAHCompressedBitmap main = new EWAHCompressedBitmap();
26 main = main.shift(64);
27 main.set(63132);
28 Assert.assertEquals((long)main.intIterator().next(),63132);
29 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
30 other.set(100);
31 other = other.shift(64);
32 other.clear(62514);
33 Assert.assertEquals((long)other.intIterator().next(),100+64);
34 }
35 @Test
36 public void issue72a() {
37 EWAHCompressedBitmap main = new EWAHCompressedBitmap();
38 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
39 main.clear(70583);
40 other = other.xor(main);
41 other.set(43013);
42 other = other.xor(main);
43 Assert.assertEquals((long)other.intIterator().next(),43013);
44 Assert.assertEquals((long)other.reverseIntIterator().next(),43013);
45 }
46 @Test
47 public void issue72b() {
48 EWAHCompressedBitmap main = new EWAHCompressedBitmap();
49 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
50 main.set(33209);
51 other = other.and(main);
52 other = other.xor(main);
53 Iterator<Integer> i = other.iterator();
54 Assert.assertEquals(i.hasNext(),true);
55 Assert.assertEquals((long)i.next(),(long)33209);
56 }
57 @Test
58 public void issue72c() {
59 EWAHCompressedBitmap main = new EWAHCompressedBitmap();
60 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
61 main = main.and(other);
62 other.clear(96836);
63 main = main.andNot(other);
64 main = main.and(other);
65 main.set(96118);
66 other = other.and(main);
67 other = other.or(main);
68 IntIterator intIterator = other.reverseIntIterator();
69 Assert.assertEquals((long)intIterator.next(),96118);
70 }
71 @Test
72 public void issue68() {
73 EWAHCompressedBitmap one = new EWAHCompressedBitmap();
74 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
75 one.set(18308);
76 other.set(24608);
77 other = other.and(one);
78 Assert.assertEquals((long)other.getFirstSetBit(),-1);
79 other.set(82764);
80 Assert.assertEquals((long)other.getFirstSetBit(),(long)other.iterator().next());
81 }
82 @Test
83 public void issue73() {
84 EWAHCompressedBitmap main = new EWAHCompressedBitmap();
85 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
86 main.clear(10684);
87 other = other.andNot(main);
88 other = other.or(main);
89 new EWAHCompressedBitmap().or(other);
90 }
91 @Test
92 public void issue74() {
93 EWAHCompressedBitmap main = new EWAHCompressedBitmap();
94 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
95 main = main.or(other);
96 other.set(7036);
97 main.set(44002);
98 other = other.and(main);
99 other = other.or(main);
100 Assert.assertEquals((long)other.iterator().next(),(long)44002);
101 }
102
103 @Test
104 public void issue70() {
105 EWAHCompressedBitmap one = new EWAHCompressedBitmap();
106 EWAHCompressedBitmap other = new EWAHCompressedBitmap();
107 one.set(16627);
108 other.set(52811);
109 other = other.and(one);
110 one = one.andNot(other);
111 one.set(16039);
112 other.set(78669);
113 other = other.or(one);
114 one = one.and(other);
115 other = other.andNot(one);
116 Assert.assertEquals((long)other.iterator().next(), 78669);
117 Assert.assertEquals((long)other.getFirstSetBit(), 78669);
118 }
119
24120 @Test
25121 public void swaptest() {
26122 EWAHCompressedBitmap x = EWAHCompressedBitmap.bitmapOf(1,2,3);
382478 public void testBug090b() throws Exception {
383479 EWAHCompressedBitmap bm1 = new EWAHCompressedBitmap();
384480 bm1.setSizeInBits(8, false); // Create a bitmap with no bit set
385 System.out.println(bm1.toDebugString());
386481 EWAHCompressedBitmap bm2 = new EWAHCompressedBitmap();
387482 bm2.setSizeInBits(64, false); // Create a bitmap with no bit set
388483 EWAHCompressedBitmap bm3 = new EWAHCompressedBitmap();
399494 public void testBug090c() throws Exception {
400495 EWAHCompressedBitmap bm1 = new EWAHCompressedBitmap();
401496 bm1.setSizeInBits(8, false); // Create a bitmap with no bit set
402 System.out.println(bm1.toDebugString());
403497 EWAHCompressedBitmap bm2 = new EWAHCompressedBitmap();
404498 bm2.setSizeInBits(64, false); // Create a bitmap with no bit set
405499 EWAHCompressedBitmap bm3 = new EWAHCompressedBitmap();
13181412 EWAHCompressedBitmap and2 = new EWAHCompressedBitmap();
13191413 FastAggregation.bufferedandWithContainer(and2, 32, bitmaps[0],bitmaps[1],bitmaps[2]);
13201414 EWAHCompressedBitmap and3 = EWAHCompressedBitmap.and(bitmaps[0],bitmaps[1],bitmaps[2]);
1321 System.out.println(and1.sizeInBits());
1322 System.out.println(and2.sizeInBits());
1323 System.out.println(and3.sizeInBits());
13241415 assertEqualsPositions(and1, and2);
13251416 assertEqualsPositions(and2, and3);
13261417 }
19222013 for (int k = 1; k < ewah.length; ++k)
19232014 answer = answer.and(ewah[k]);
19242015 // result should be empty
1925 if (answer.toList().size() != 0)
1926 System.out.println(answer.toDebugString());
19272016 Assert.assertTrue(answer.toList().size() == 0);
19282017 Assert.assertTrue(EWAHCompressedBitmap.and(ewah).toList()
19292018 .size() == 0);
19892078 EWAHCompressedBitmap.or(ewah));
19902079 int k = 0;
19912080 for (int j : answer) {
1992 if (k != j)
1993 System.out.println(answer
1994 .toDebugString());
19952081 Assert.assertEquals(k, j);
19962082 k += 1;
19972083 }
20502136 }
20512137 int k = 0;
20522138 for (int j : answer) {
2053 if (k != j)
2054 System.out.println(answer.toDebugString());
20552139 Assert.assertEquals(k, j);
20562140 k += 1;
20572141 }
0 package com.googlecode.javaewah;
1
2 import java.util.BitSet;
3 import java.util.Iterator;
4 // credit @svanmald
5 public class ForwardBitSetIterator implements Iterator<Integer> {
6
7 private final BitSet bitSet;
8 private int next;
9
10 public ForwardBitSetIterator(BitSet bitSet) {
11 this.bitSet = bitSet;
12 this.next = bitSet.nextSetBit(0);
13 }
14
15
16 @Override
17 public boolean hasNext() {
18 return next != -1;
19 }
20
21 @Override
22 public Integer next() {
23 int current = next;
24 next = bitSet.nextSetBit(next + 1);
25 return current;
26 }
27 }
0 package com.googlecode.javaewah;
1
2 import java.util.Random;
3 import org.junit.Test;
4 // credit @svanmald
5 @SuppressWarnings("javadoc")
6 public class FuzzEWAHTest {
7 public static boolean areAssertsEnabled() {
8 boolean assertsEnabled = false;
9 assert assertsEnabled = true; // Intentional side effect!!!
10 return assertsEnabled;
11 }
12
13 @Test
14 public void testEwah() {
15 if(!areAssertsEnabled()) { throw new RuntimeException("asserts need to be enabled."); }
16 // ENABLE ASSERTS BEFORE EXECUTING TO ENABLE VALIDATION.
17 // if print = false and seed -1, the code will execute 10 random mutation to 2 bitmaps until infinity and each time validate the result
18 // Each time a set of 10 random mutations starts, a seed is printed (even if print = false).
19 // if one of the sets of 10 mutations fails validation, the printed seed can be used here together with print = true to reproduce the issue
20 System.out.println(" == Launching @svanmald's fuzzer! ");
21 testEwah(false, -1);
22 }
23
24 private void testEwah(boolean print, int seed) {
25 Random seedGenerator = new Random();
26 Mutation[] mutations = Mutation.values();
27 int times = 1000000;
28
29 while (times > 0) {
30 times --;
31 if((times % 10000) == 0) { System.out.print("."); System.out.flush(); }
32 int currentSeed = seed;
33 if (currentSeed == -1) {
34 currentSeed = seedGenerator.nextInt();
35 }
36 if (print) {
37 System.out.println("Seed " + currentSeed);
38 }
39 Random seededRandom = new Random(currentSeed);
40 EWAHBitSetPair main = new EWAHBitSetPair();
41 if (print) {
42 System.out.println("EWAHCompressedBitmap main = new EWAHCompressedBitmap();");
43 }
44 EWAHBitSetPair other = new EWAHBitSetPair();
45 if (print) {
46 System.out.println("EWAHCompressedBitmap other = new EWAHCompressedBitmap();");
47 }
48 for (int i = 0; i < 10; i++) {
49 Mutation mutation = mutations[seededRandom.nextInt(mutations.length)];
50 mutation.apply(print, seededRandom, main, other);
51 main.validate();
52 other.validate();
53 }
54 }
55 System.out.println();
56 }
57
58 public enum Mutation {
59 OR {
60 @Override
61 void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other) {
62 if (random.nextDouble() < 0.5) {
63 if (print) {
64 System.out.println("main = main.or(other);");
65 }
66 main.or(other);
67 } else {
68 if (print) {
69 System.out.println("other = other.or(main);");
70 }
71 other.or(main);
72 }
73 }
74 },
75 AND {
76 @Override
77 void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other) {
78 if (random.nextDouble() < 0.5) {
79 if (print) {
80 System.out.println("main = main.and(other);");
81 }
82 main.and(other);
83 } else {
84 if (print) {
85 System.out.println("other = other.and(main);");
86 }
87 other.and(main);
88 }
89 }
90 },
91 AND_NOT {
92 @Override
93 void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other) {
94 if (random.nextDouble() < 0.5) {
95 if (print) {
96 System.out.println("main = main.andNot(other);");
97 }
98 main.andNot(other);
99 } else {
100 if (print) {
101 System.out.println("other = other.andNot(main);");
102 }
103 other.andNot(main);
104 }
105 }
106 },
107 XOR {
108 @Override
109 void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other) {
110 if (random.nextDouble() < 0.5) {
111 if (print) {
112 System.out.println("main = main.xor(other);");
113 }
114 main.xor(other);
115 } else {
116 if (print) {
117 System.out.println("other = other.xor(main);");
118 }
119 other.xor(main);
120 }
121 }
122 },
123 SET {
124 @Override
125 void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other) {
126 int value = random.nextInt(100_000);
127 if (random.nextDouble() < 0.5) {
128 if (print) {
129 System.out.println("main.set(" + value + ");");
130 }
131 main.set(value);
132 } else {
133 if (print) {
134 System.out.println("other.set(" + value + ");");
135 }
136 other.set(value);
137 }
138 }
139 },
140 CLEAR_RANDOM {
141 @Override
142 void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other) {
143 int value = random.nextInt(100_000);
144 if (random.nextDouble() < 0.5) {
145 if (print) {
146 System.out.println("main.clear(" + value + ");");
147 }
148 main.clear(value);
149 } else {
150 if (print) {
151 System.out.println("other.clear(" + value + ");");
152 }
153 other.clear(value);
154 }
155 }
156 };
157
158 abstract void apply(boolean print, Random random, EWAHBitSetPair main, EWAHBitSetPair other);
159 }
160 }
0 package com.googlecode.javaewah32;
1
2 import com.googlecode.javaewah.EWAHCompressedBitmap;
3 import com.googlecode.javaewah.IntIterator;
4 import java.util.BitSet;
5 import com.googlecode.javaewah.BackwardBitSetIterator;
6 import com.googlecode.javaewah.ForwardBitSetIterator;
7 // credit @svanmald
8 public class EWAH32BitSetPair {
9
10 private EWAHCompressedBitmap bitmap;
11 private BitSet bitSet;
12
13 public EWAH32BitSetPair() {
14 bitmap = new EWAHCompressedBitmap();
15 bitSet = new BitSet();
16 }
17
18 public void validate() {
19 assert bitmap.cardinality() == bitSet.cardinality();
20 ForwardBitSetIterator forwardBitSetIterator = new ForwardBitSetIterator(bitSet);
21 for (Integer current : bitmap) {
22 Integer next = forwardBitSetIterator.next();
23 assert bitmap.get(current);
24 assert next.equals(current);
25 }
26
27 BackwardBitSetIterator backwardBitSetIterator = new BackwardBitSetIterator(bitSet);
28 IntIterator reverseIterator = bitmap.reverseIntIterator();
29 while (reverseIterator.hasNext()) {
30 int nextBitMap = reverseIterator.next();
31 Integer nextBitSet = backwardBitSetIterator.next();
32 assert nextBitSet == nextBitMap;
33 }
34 assert !backwardBitSetIterator.hasNext();
35
36 EWAHCompressedBitmap result = new EWAHCompressedBitmap().or(bitmap);
37 assert result.equals(bitmap);
38 assert bitmap.equals(result);
39 assert bitmap.isEmpty() || bitmap.getFirstSetBit() == bitmap.iterator().next();
40 }
41
42 public void or(EWAH32BitSetPair other) {
43 bitmap = bitmap.or(other.bitmap);
44 bitSet.or(other.bitSet);
45 }
46
47 public void and(EWAH32BitSetPair other) {
48 bitmap = bitmap.and(other.bitmap);
49 bitSet.and(other.bitSet);
50 }
51
52 public void andNot(EWAH32BitSetPair other) {
53 bitmap = bitmap.andNot(other.bitmap);
54 bitSet.andNot(other.bitSet);
55 }
56
57 public void xor(EWAH32BitSetPair other) {
58 bitmap = bitmap.xor(other.bitmap);
59 bitSet.xor(other.bitSet);
60 }
61
62 public void set(int value) {
63 bitSet.set(value);
64 bitmap.set(value);
65 }
66
67 public void clear(int value) {
68 bitSet.clear(value);
69 bitmap.clear(value);
70 }
71 }
2222 */
2323 @SuppressWarnings("javadoc")
2424 public class EWAHCompressedBitmap32Test {
25
25 @Test
26 public void issue77() {
27 EWAHCompressedBitmap32 main = new EWAHCompressedBitmap32();
28 main = main.shift(64);
29 main.set(63132);
30 Assert.assertEquals((long)main.intIterator().next(),63132);
31 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
32 other.set(100);
33 other = other.shift(64);
34 other.clear(62514);
35 Assert.assertEquals((long)other.intIterator().next(),100+64);
36 }
37 @Test
38 public void issue72a() {
39 EWAHCompressedBitmap32 main = new EWAHCompressedBitmap32();
40 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
41 main.clear(70583);
42 other = other.xor(main);
43 other.set(43013);
44 other = other.xor(main);
45 Assert.assertEquals((long)other.intIterator().next(),43013);
46 Assert.assertEquals((long)other.reverseIntIterator().next(),43013);
47 }
48 @Test
49 public void issue72b() {
50 EWAHCompressedBitmap32 main = new EWAHCompressedBitmap32();
51 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
52 main.set(33209);
53 other = other.and(main);
54 other = other.xor(main);
55 System.out.println(other);
56 Iterator<Integer> i = other.iterator();
57 Assert.assertEquals(i.hasNext(),true);
58 Assert.assertEquals((long)i.next(),(long)33209);
59 }
60 @Test
61 public void issue72c() {
62 EWAHCompressedBitmap32 main = new EWAHCompressedBitmap32();
63 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
64 main = main.and(other);
65 other.clear(96836);
66 main = main.andNot(other);
67 main = main.and(other);
68 main.set(96118);
69 other = other.and(main);
70 other = other.or(main);
71 System.out.println(other);
72 IntIterator intIterator = other.reverseIntIterator();
73 Assert.assertEquals((long)intIterator.next(),96118);
74 }
75
76 @Test
77 public void issue73() {
78 EWAHCompressedBitmap32 main = new EWAHCompressedBitmap32();
79 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
80 main.clear(10684);
81 other = other.andNot(main);
82 other = other.or(main);
83 new EWAHCompressedBitmap32().or(other);
84 }
85
86 @Test
87 public void issue74() {
88 EWAHCompressedBitmap32 main = new EWAHCompressedBitmap32();
89 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
90 main = main.or(other);
91 other.set(7036);
92 main.set(44002);
93 other = other.and(main);
94 other = other.or(main);
95 Assert.assertEquals((long)other.iterator().next(),(long)44002);
96 }
97
98 @Test
99 public void issue68() {
100 EWAHCompressedBitmap32 one = new EWAHCompressedBitmap32();
101 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
102 one.set(18308);
103 other.set(24608);
104 other = other.and(one);
105 Assert.assertEquals((long)other.getFirstSetBit(),-1);
106 other.set(82764);
107 Assert.assertEquals((long)other.getFirstSetBit(),(long)other.iterator().next());
108 }
109
110 @Test
111 public void issue70() {
112 EWAHCompressedBitmap32 one = new EWAHCompressedBitmap32();
113 EWAHCompressedBitmap32 other = new EWAHCompressedBitmap32();
114 one.set(16627);
115 other.set(52811);
116 other = other.and(one);
117 one = one.andNot(other);
118 one.set(16039);
119 other.set(78669);
120 other = other.or(one);
121 one = one.and(other);
122 other = other.andNot(one);
123 Assert.assertEquals((long)other.iterator().next(), 78669);
124 Assert.assertEquals((long)other.getFirstSetBit(), 78669);
125 }
126
26127 @Test
27128 public void swaptest() {
28129 EWAHCompressedBitmap32 x = EWAHCompressedBitmap32.bitmapOf(1,2,3);
327428 public void testBug090b() throws Exception {
328429 EWAHCompressedBitmap32 bm1 = new EWAHCompressedBitmap32();
329430 bm1.setSizeInBits(8, false); // Create a bitmap with no bit set
330 System.out.println(bm1.toDebugString());
331431 EWAHCompressedBitmap32 bm2 = new EWAHCompressedBitmap32();
332432 bm2.setSizeInBits(32, false); // Create a bitmap with no bit set
333433 EWAHCompressedBitmap32 bm3 = new EWAHCompressedBitmap32();
372472 public void testBug090c() throws Exception {
373473 EWAHCompressedBitmap32 bm1 = new EWAHCompressedBitmap32();
374474 bm1.setSizeInBits(8, false); // Create a bitmap with no bit set
375 System.out.println(bm1.toDebugString());
376475 EWAHCompressedBitmap32 bm2 = new EWAHCompressedBitmap32();
377476 bm2.setSizeInBits(64, false); // Create a bitmap with no bit set
378477 EWAHCompressedBitmap32 bm3 = new EWAHCompressedBitmap32();
12341333 EWAHCompressedBitmap32 and2 = new EWAHCompressedBitmap32();
12351334 FastAggregation32.bufferedandWithContainer(and2, 32, bitmaps[0],bitmaps[1],bitmaps[2]);
12361335 EWAHCompressedBitmap32 and3 = EWAHCompressedBitmap32.and(bitmaps[0],bitmaps[1],bitmaps[2]);
1237 System.out.println(and1.sizeInBits());
1238 System.out.println(and2.sizeInBits());
1239 System.out.println(and3.sizeInBits());
12401336 assertEqualsPositions(and1, and2);
12411337 assertEqualsPositions(and2, and3);
12421338 }
18411937 for (int k = 1; k < ewah.length; ++k)
18421938 answer = answer.and(ewah[k]);
18431939 // result should be empty
1844 if (answer.toList().size() != 0)
1845 System.out.println(answer.toDebugString());
18461940 Assert.assertTrue(answer.toList().size() == 0);
18471941 Assert.assertTrue(EWAHCompressedBitmap32.and(ewah)
18481942 .toList().size() == 0);
19332027 EWAHCompressedBitmap32.or(ewah));
19342028 int k = 0;
19352029 for (int j : answer) {
1936 if (k != j)
1937 System.out.println(answer
1938 .toDebugString());
19392030 Assert.assertEquals(k, j);
19402031 k += 1;
19412032 }
19692060 }
19702061 int k = 0;
19712062 for (int j : answer) {
1972 if (k != j)
1973 System.out.println(answer.toDebugString());
19742063 Assert.assertEquals(k, j);
19752064 k += 1;
19762065 }
0 package com.googlecode.javaewah32;
1
2 import java.util.Random;
3 import org.junit.Test;
4 // credit @svanmald
5 @SuppressWarnings("javadoc")
6 public class FuzzEWAH32Test {
7 public static boolean areAssertsEnabled() {
8 boolean assertsEnabled = false;
9 assert assertsEnabled = true; // Intentional side effect!!!
10 return assertsEnabled;
11 }
12
13 @Test
14 public void testEwah() {
15 if(!areAssertsEnabled()) { throw new RuntimeException("asserts need to be enabled."); }
16 // ENABLE ASSERTS BEFORE EXECUTING TO ENABLE VALIDATION.
17 // if print = false and seed -1, the code will execute 10 random mutation to 2 bitmaps until infinity and each time validate the result
18 // Each time a set of 10 random mutations starts, a seed is printed (even if print = false).
19 // if one of the sets of 10 mutations fails validation, the printed seed can be used here together with print = true to reproduce the issue
20 System.out.println(" == Launching @svanmald's fuzzer! ");
21 testEwah(false, -1);
22 }
23
24 private void testEwah(boolean print, int seed) {
25 Random seedGenerator = new Random();
26 Mutation[] mutations = Mutation.values();
27 int times = 1000000;
28
29 while (times > 0) {
30 times --;
31 if((times % 10000) == 0) { System.out.print("."); System.out.flush(); }
32 int currentSeed = seed;
33 if (currentSeed == -1) {
34 currentSeed = seedGenerator.nextInt();
35 }
36 if (print) {
37 System.out.println("Seed " + currentSeed);
38 }
39 Random seededRandom = new Random(currentSeed);
40 EWAH32BitSetPair main = new EWAH32BitSetPair();
41 if (print) {
42 System.out.println("EWAHCompressedBitmap main = new EWAHCompressedBitmap();");
43 }
44 EWAH32BitSetPair other = new EWAH32BitSetPair();
45 if (print) {
46 System.out.println("EWAHCompressedBitmap other = new EWAHCompressedBitmap();");
47 }
48 for (int i = 0; i < 10; i++) {
49 Mutation mutation = mutations[seededRandom.nextInt(mutations.length)];
50 mutation.apply(print, seededRandom, main, other);
51 main.validate();
52 other.validate();
53 }
54 }
55 System.out.println();
56 }
57
58 public enum Mutation {
59 OR {
60 @Override
61 void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other) {
62 if (random.nextDouble() < 0.5) {
63 if (print) {
64 System.out.println("main = main.or(other);");
65 }
66 main.or(other);
67 } else {
68 if (print) {
69 System.out.println("other = other.or(main);");
70 }
71 other.or(main);
72 }
73 }
74 },
75 AND {
76 @Override
77 void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other) {
78 if (random.nextDouble() < 0.5) {
79 if (print) {
80 System.out.println("main = main.and(other);");
81 }
82 main.and(other);
83 } else {
84 if (print) {
85 System.out.println("other = other.and(main);");
86 }
87 other.and(main);
88 }
89 }
90 },
91 AND_NOT {
92 @Override
93 void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other) {
94 if (random.nextDouble() < 0.5) {
95 if (print) {
96 System.out.println("main = main.andNot(other);");
97 }
98 main.andNot(other);
99 } else {
100 if (print) {
101 System.out.println("other = other.andNot(main);");
102 }
103 other.andNot(main);
104 }
105 }
106 },
107 XOR {
108 @Override
109 void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other) {
110 if (random.nextDouble() < 0.5) {
111 if (print) {
112 System.out.println("main = main.xor(other);");
113 }
114 main.xor(other);
115 } else {
116 if (print) {
117 System.out.println("other = other.xor(main);");
118 }
119 other.xor(main);
120 }
121 }
122 },
123 SET {
124 @Override
125 void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other) {
126 int value = random.nextInt(100_000);
127 if (random.nextDouble() < 0.5) {
128 if (print) {
129 System.out.println("main.set(" + value + ");");
130 }
131 main.set(value);
132 } else {
133 if (print) {
134 System.out.println("other.set(" + value + ");");
135 }
136 other.set(value);
137 }
138 }
139 },
140 CLEAR_RANDOM {
141 @Override
142 void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other) {
143 int value = random.nextInt(100_000);
144 if (random.nextDouble() < 0.5) {
145 if (print) {
146 System.out.println("main.clear(" + value + ");");
147 }
148 main.clear(value);
149 } else {
150 if (print) {
151 System.out.println("other.clear(" + value + ");");
152 }
153 other.clear(value);
154 }
155 }
156 };
157
158 abstract void apply(boolean print, Random random, EWAH32BitSetPair main, EWAH32BitSetPair other);
159 }
160 }