From 9d8369660f433d7eb82b8596ec9062cf6c8ddfea Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Mon, 27 Feb 2017 14:00:22 -0800 Subject: [PATCH 1/9] Fixed equality in BufferedString. A BufferedString cannot be equal to a java String, since this breaks (super)symmetry. --- h2o-core/src/main/java/water/H2O.java | 2 +- h2o-core/src/main/java/water/H2ONode.java | 1 + h2o-core/src/main/java/water/HeartBeatThread.java | 1 + .../src/main/java/water/parser/BufferedString.java | 26 ++- .../src/main/java/water/parser/Categorical.java | 2 +- h2o-core/src/main/java/water/parser/CsvParser.java | 14 +- .../test/java/water/parser/BufferedStringTest.java | 189 +++++++++++++++++++++ 7 files changed, 215 insertions(+), 20 deletions(-) create mode 100644 h2o-core/src/test/java/water/parser/BufferedStringTest.java diff --git a/h2o-core/src/main/java/water/H2O.java b/h2o-core/src/main/java/water/H2O.java index 77b9ab17b76..c30371f18b6 100644 --- a/h2o-core/src/main/java/water/H2O.java +++ b/h2o-core/src/main/java/water/H2O.java @@ -1535,7 +1535,7 @@ static boolean larger( int nnn, int old ) { } public final int size() { return _memary.length; } - final H2ONode leader() { return _memary[0]; } + final H2ONode leader() { return _memary == null || size() == 0 ? null : _memary[0]; } // Find the node index for this H2ONode, or a negative number on a miss int nidx( H2ONode h2o ) { return java.util.Arrays.binarySearch(_memary,h2o); } diff --git a/h2o-core/src/main/java/water/H2ONode.java b/h2o-core/src/main/java/water/H2ONode.java index c966797f690..8e3ccc0094f 100644 --- a/h2o-core/src/main/java/water/H2ONode.java +++ b/h2o-core/src/main/java/water/H2ONode.java @@ -398,6 +398,7 @@ public void sendMessage(ByteBuffer bb, byte msg_priority) { bb = _msgQ.poll(); // Go get more, same batch } sendBuffer(); // Send final trailing BBs + } catch (IllegalMonitorStateException imse) { /* ignore */ } catch (InterruptedException e) { /*ignore*/ } } } catch(Throwable t) { throw Log.throwErr(t); } diff --git a/h2o-core/src/main/java/water/HeartBeatThread.java b/h2o-core/src/main/java/water/HeartBeatThread.java index f9be9ab0c75..1dfce7d3dc3 100644 --- a/h2o-core/src/main/java/water/HeartBeatThread.java +++ b/h2o-core/src/main/java/water/HeartBeatThread.java @@ -165,6 +165,7 @@ public void run() { // Once per second, for the entire cloud a Node will multi-cast publish // itself, so other unrelated Clouds discover each other and form up. try { Thread.sleep(SLEEP); } // Only once-sec per entire Cloud + catch( IllegalMonitorStateException ignore ) { } catch( InterruptedException ignore ) { } } } diff --git a/h2o-core/src/main/java/water/parser/BufferedString.java b/h2o-core/src/main/java/water/parser/BufferedString.java index 8395fe45fe6..98d5903fa8e 100644 --- a/h2o-core/src/main/java/water/parser/BufferedString.java +++ b/h2o-core/src/main/java/water/parser/BufferedString.java @@ -61,6 +61,7 @@ public final BufferedString read_impl(AutoBuffer ab){ return hash; } + // TODO(vlad): make sure that this method is not as destructive as it now is (see tests) void addChar() { _len++; } @@ -150,17 +151,24 @@ public void setOff(int off) { for (int i = 0; i < _len; ++i) if (_buf[_off + i] != str._buf[str._off + i]) return false; return true; - } // FIXME: Called in NA_String detection during CsvParser, UTF-8 sensitive - else if (o instanceof String) { - String str = (String) o; - if (str.length() != _len) return false; - for (int i = 0; i < _len; ++i) - if (_buf[_off + i] != str.charAt(i)) return false; - return true; } - return false; //FIXME find out if this is required for some case or if an exception can be thrown + return false; } - + + public boolean sameString(String str) { + if (str == null || str.length() != _len) return false; + for (int i = 0; i < _len; ++i) + if ((0xFF&_buf[_off + i]) != str.charAt(i)) return false; + return true; + } + + public boolean isOneOf(String[] samples) { + if (samples != null) { + for (String sample : samples) if (sameString(sample)) return true; + } + return false; + } + // Thou Shalt Not use accessors in performance critical code - because it // obfuscates the code's cost model. All file-local uses of the accessors // has been stripped, please do not re-insert them. In particular, the diff --git a/h2o-core/src/main/java/water/parser/Categorical.java b/h2o-core/src/main/java/water/parser/Categorical.java index 4fdc486a50c..8e228828273 100644 --- a/h2o-core/src/main/java/water/parser/Categorical.java +++ b/h2o-core/src/main/java/water/parser/Categorical.java @@ -67,7 +67,7 @@ public void convertToUTF8(int col){ StringBuilder hexSB = new StringBuilder(); for (int i =0; i < bStrs.length; i++) { String s = bStrs[i].toString(); - if (!bStrs[i].equals(s)) { + if (!bStrs[i].sameString(s)) { if (s.contains("\uFFFD")) { // make weird chars into hex s = bStrs[i].bytesToString(); if (hexConvCnt++ < MAX_EXAMPLES) hexSB.append(s +", "); diff --git a/h2o-core/src/main/java/water/parser/CsvParser.java b/h2o-core/src/main/java/water/parser/CsvParser.java index 63939be13a6..648ac14c7bd 100644 --- a/h2o-core/src/main/java/water/parser/CsvParser.java +++ b/h2o-core/src/main/java/water/parser/CsvParser.java @@ -119,15 +119,11 @@ assert str.getBuffer() != bits; str.addBuff(bits); } - if( _setup._na_strings != null - && _setup._na_strings.length > colIdx - && _setup._na_strings[colIdx] != null) { - for (String s : _setup._na_strings[colIdx]) { - if (str.equals(s)) { - isNa = true; - break; - } - } + if( !isNa && + _setup._na_strings != null && + _setup._na_strings.length > colIdx && + str.isOneOf(_setup._na_strings[colIdx])) { + isNa = true; } if (!isNa) { dout.addStrCol(colIdx, str); diff --git a/h2o-core/src/test/java/water/parser/BufferedStringTest.java b/h2o-core/src/test/java/water/parser/BufferedStringTest.java new file mode 100644 index 00000000000..e3a5109e66c --- /dev/null +++ b/h2o-core/src/test/java/water/parser/BufferedStringTest.java @@ -0,0 +1,189 @@ +package water.parser; + +import org.junit.Ignore; +import org.junit.Test; +import water.AutoBuffer; +import water.Paxos; +import water.TestUtil; + +import static org.junit.Assert.*; + +/** + * This is mostly a skeleton of the tests, feel free to implement the cases + * Created by vpatryshev on 1/17/17. + */ +public class BufferedStringTest { + + @Test + public void testWrite_impl() throws Exception { + final String source = "this is not a string"; + BufferedString sut = new BufferedString(source); + assertArrayEquals(source.getBytes(), sut.getBuffer()); + AutoBuffer ab = new AutoBuffer(); + sut.write_impl(ab); + final byte[] expected = ("\u0015" + source).getBytes(); + final byte[] actual = ab.buf(); + assertArrayEquals(expected, actual); + } + + @Test + public void testRead_impl() throws Exception { + final String source = "this is not a string"; + BufferedString sut1 = new BufferedString(source); + AutoBuffer ab = new AutoBuffer(); + sut1.write_impl(ab); + ab.bufClose(); +// BufferedString sut2 = new BufferedString("what?"); +// sut2.read_impl(ab); +// assertEquals(sut1, sut2); + } + + @Test + public void testCompareTo() throws Exception { + final String source = "this is not a string"; + BufferedString sut1 = new BufferedString(source); + assertEquals(0, sut1.compareTo(new BufferedString(source))); + assertEquals(2, sut1.compareTo(new BufferedString("this is not a stri"))); + } + + @Test + public void testHashCode() throws Exception { + + } + + @Test + public void testAddChar() throws Exception { + final String source = "abc"; + BufferedString sut1 = new BufferedString(source); + assertEquals(3, sut1.length()); + sut1.addChar(); + assertEquals(4, sut1.length()); +// TODO(vlad): fix the crash in the next line +// String actual = sut1.bytesToString(); +// assertEquals(source, actual); + // TODO(vlad): fix it; we don't need the cloud +// Paxos._commonKnowledge = true; // this is totally stupid; thank you Cliff for the fun +// TODO(vlad): fix the crash in the next line +// byte[] bytes = sut1.asBytes(); +// assertArrayEquals(source.getBytes(), bytes); + } + + @Test + public void testAddBuff() throws Exception { + + } + + @Test + public void testToString() throws Exception { + + } + + @Test + public void testBytesToString() throws Exception { + + } + + @Test + public void testToString1() throws Exception { + + } + + @Test + public void testToBufferedString() throws Exception { + + } + + @Test + public void testSet() throws Exception { + + } + + @Test + public void testSet1() throws Exception { + + } + + @Test + public void testSet2() throws Exception { + + } + + @Test + public void testSetOff() throws Exception { + + } + + @Test + public void testEquals() throws Exception { + BufferedString sut = new BufferedString("abc"); + assertEquals(sut, sut); + assertEquals(sut, new BufferedString("abc")); + assertFalse(sut.equals("abc")); + assertFalse(sut.equals(new BufferedString("abcd"))); + assertFalse(sut.equals(new BufferedString("ABCD"))); + assertFalse(sut.equals(new BufferedString(" abc"))); + assertFalse(sut.equals(new BufferedString("abc "))); + assertFalse(sut.equals(new BufferedString("abc\0"))); + assertFalse(sut.equals(new BufferedString("ab"))); + assertFalse(sut.equals(new BufferedString(""))); + assertFalse(new BufferedString("").equals(sut)); + } + + @Test + public void testSameString() throws Exception { + BufferedString sut1 = new BufferedString("abc"); + assertFalse(sut1.sameString(null)); + assertTrue(sut1.sameString("abc")); + assertFalse(sut1.sameString("ab")); + assertFalse(sut1.sameString("abd")); + assertFalse(sut1.sameString("abcd")); + assertFalse(sut1.sameString("abC")); + assertFalse(sut1.sameString("ab\u0441")); // this is Russian 'c' here + assertFalse(sut1.sameString("ab")); + BufferedString sut2 = new BufferedString(""); + assertTrue(sut2.sameString("")); + assertFalse(sut1.sameString(null)); + assertFalse(sut2.sameString("a")); + BufferedString sut3 = new BufferedString("a\u0100b"); + assertFalse(sut3.sameString("a\u0100b")); + } + + @Ignore("This test is failing because the method is wrong and must be fixed, see PUBDEV-3957") + @Test public void testSameStringUTF8() throws Exception { + BufferedString sut4 = new BufferedString("a\u0088b"); + assertTrue(sut4.sameString("a\u0088b")); + } + + @Test public void testIsOneOf() throws Exception { + BufferedString sut = new BufferedString("abc"); + assertFalse(sut.isOneOf(null)); + assertFalse(sut.isOneOf(new String[]{})); + assertFalse(sut.isOneOf(new String[]{"", "a", "b", "ab", "bc", "abcd", "xabc"})); + assertTrue(sut.isOneOf(new String[]{"abc", "a", "b", "ab", "bc", "abcd"})); + assertTrue(sut.isOneOf(new String[]{"a", "b", "ab", "bc", "abcd", "abc"})); + assertTrue(sut.isOneOf(new String[]{"", "b", "ab", "bc", "abcd", "abc", "whateva"})); + assertTrue(sut.isOneOf(new String[]{"", null, "ab", "bc", "abcd", "abc", "whateva"})); + } + + @Test + public void testGetBuffer() throws Exception { + final String source = "this is not a string\u00f0"; + BufferedString sut = new BufferedString(source); + assertArrayEquals(source.getBytes(), sut.getBuffer()); + } + + @Test + public void testGetOffset() throws Exception { + + } + + @Test + public void testLength() throws Exception { + + } + + @Test + public void testGetNumericType() throws Exception { + + } +} \ No newline at end of file From af8a2007dc0b0229b1faa593489554eedf1f3824 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Mon, 27 Feb 2017 22:58:03 -0800 Subject: [PATCH 2/9] Fixed some failing tests; accounted for Michal's remarks. --- h2o-core/src/main/java/water/H2O.java | 6 +++++- h2o-core/src/test/java/water/fvec/CStrChunkTest.java | 4 ++-- .../java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/h2o-core/src/main/java/water/H2O.java b/h2o-core/src/main/java/water/H2O.java index c30371f18b6..2dfea80ac12 100644 --- a/h2o-core/src/main/java/water/H2O.java +++ b/h2o-core/src/main/java/water/H2O.java @@ -1535,7 +1535,11 @@ static boolean larger( int nnn, int old ) { } public final int size() { return _memary.length; } - final H2ONode leader() { return _memary == null || size() == 0 ? null : _memary[0]; } + final H2ONode leader() { +// assert _memary != null; +// assert _memary.length > 0; + return _memary[0]; + } // Find the node index for this H2ONode, or a negative number on a miss int nidx( H2ONode h2o ) { return java.util.Arrays.binarySearch(_memary,h2o); } diff --git a/h2o-core/src/test/java/water/fvec/CStrChunkTest.java b/h2o-core/src/test/java/water/fvec/CStrChunkTest.java index 4cb5b11aea8..b6f1f84d0e4 100644 --- a/h2o-core/src/test/java/water/fvec/CStrChunkTest.java +++ b/h2o-core/src/test/java/water/fvec/CStrChunkTest.java @@ -101,9 +101,9 @@ public void test_sparse() { nc.addStr(new BufferedString("bar")); Chunk c = nc.compress(); Assert.assertTrue("first 100 entries are NA",c.isNA(0) && c.isNA(99)); - Assert.assertTrue("Sparse string has values",c.atStr(new BufferedString(),100).equals("foo")); + Assert.assertTrue("Sparse string has values",c.atStr(new BufferedString(),100).sameString("foo")); Assert.assertTrue("NA",c.isNA(101)); - Assert.assertTrue("Sparse string has values",c.atStr(new BufferedString(),102).equals("bar")); + Assert.assertTrue("Sparse string has values",c.atStr(new BufferedString(),102).sameString("bar")); } } diff --git a/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java b/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java index 964d099d5dc..239a1b1e60d 100644 --- a/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java +++ b/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java @@ -42,6 +42,6 @@ Val res2 = Rapids.exec("(isax " + fr1._key + " 10 10 0)"); // 10 words 10 max cardinality 0 optimize card fr2 = res2.getFrame(); String isaxIDX = "0^10_0^10_0^10_0^10_5^10_7^10_8^10_9^10_9^10_8^10"; - Assert.assertEquals(fr2.vec(0).atStr(new BufferedString(),0),isaxIDX); + Assert.assertEquals(fr2.vec(0).atStr(new BufferedString(),0).toString(),isaxIDX); } } From 33c73b931aa40d34a93250233b3237e92026927a Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Tue, 28 Feb 2017 10:48:06 -0800 Subject: [PATCH 3/9] Added more output for failing(?) tests. --- h2o-core/src/test/java/water/fvec/CStrChunkTest.java | 3 ++- h2o-core/src/test/java/water/parser/BufferedStringTest.java | 12 ++++++++++-- .../water/rapids/ast/prims/timeseries/TimeSeriesTests.java | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/h2o-core/src/test/java/water/fvec/CStrChunkTest.java b/h2o-core/src/test/java/water/fvec/CStrChunkTest.java index b6f1f84d0e4..2974d18eeed 100644 --- a/h2o-core/src/test/java/water/fvec/CStrChunkTest.java +++ b/h2o-core/src/test/java/water/fvec/CStrChunkTest.java @@ -103,7 +103,8 @@ public void test_sparse() { Assert.assertTrue("first 100 entries are NA",c.isNA(0) && c.isNA(99)); Assert.assertTrue("Sparse string has values",c.atStr(new BufferedString(),100).sameString("foo")); Assert.assertTrue("NA",c.isNA(101)); - Assert.assertTrue("Sparse string has values",c.atStr(new BufferedString(),102).sameString("bar")); + final BufferedString bufferedString = c.atStr(new BufferedString(), 102); + Assert.assertTrue("Sparse string has values: expected `bar`, got " + bufferedString, bufferedString.sameString("bar")); } } diff --git a/h2o-core/src/test/java/water/parser/BufferedStringTest.java b/h2o-core/src/test/java/water/parser/BufferedStringTest.java index e3a5109e66c..86542d45c69 100644 --- a/h2o-core/src/test/java/water/parser/BufferedStringTest.java +++ b/h2o-core/src/test/java/water/parser/BufferedStringTest.java @@ -6,6 +6,10 @@ import water.Paxos; import water.TestUtil; +import java.util.Arrays; + +import static java.util.Arrays.*; + import static org.junit.Assert.*; /** @@ -167,9 +171,13 @@ public void testSameString() throws Exception { @Test public void testGetBuffer() throws Exception { - final String source = "this is not a string\u00f0"; + final String source = "not a string\u00f0"; BufferedString sut = new BufferedString(source); - assertArrayEquals(source.getBytes(), sut.getBuffer()); + final byte[] expected = source.getBytes(); + final byte[] actual = sut.getBuffer(); + assertArrayEquals("Failed. expected " + Arrays.toString(expected) + + ", got " + Arrays.toString(actual), + expected, actual); } @Test diff --git a/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java b/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java index 239a1b1e60d..96e272832fa 100644 --- a/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java +++ b/h2o-core/src/test/java/water/rapids/ast/prims/timeseries/TimeSeriesTests.java @@ -41,7 +41,8 @@ DKV.put(fr1); Val res2 = Rapids.exec("(isax " + fr1._key + " 10 10 0)"); // 10 words 10 max cardinality 0 optimize card fr2 = res2.getFrame(); - String isaxIDX = "0^10_0^10_0^10_0^10_5^10_7^10_8^10_9^10_9^10_8^10"; - Assert.assertEquals(fr2.vec(0).atStr(new BufferedString(),0).toString(),isaxIDX); + String expected = "0^10_0^10_0^10_0^10_5^10_7^10_8^10_9^10_9^10_8^10"; + final String actual = fr2.vec(0).atStr(new BufferedString(), 0).toString(); + Assert.assertEquals(expected, actual); } } From 0398882210a35c74ca33dc3f7134017b303e5e49 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Wed, 1 Mar 2017 11:36:17 -0800 Subject: [PATCH 4/9] Fixed the test; default --- h2o-core/src/test/java/water/parser/BufferedStringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h2o-core/src/test/java/water/parser/BufferedStringTest.java b/h2o-core/src/test/java/water/parser/BufferedStringTest.java index 86542d45c69..107d329b630 100644 --- a/h2o-core/src/test/java/water/parser/BufferedStringTest.java +++ b/h2o-core/src/test/java/water/parser/BufferedStringTest.java @@ -173,7 +173,7 @@ public void testSameString() throws Exception { public void testGetBuffer() throws Exception { final String source = "not a string\u00f0"; BufferedString sut = new BufferedString(source); - final byte[] expected = source.getBytes(); + final byte[] expected = source.getBytes("UTF8"); final byte[] actual = sut.getBuffer(); assertArrayEquals("Failed. expected " + Arrays.toString(expected) + ", got " + Arrays.toString(actual), From 60d6f40ad02eb10cb3ac96b2d2aa9cc26a3fd9b3 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Tue, 7 Mar 2017 16:18:43 -0800 Subject: [PATCH 5/9] Added @Ignore, on Michal's request --- .../test/java/water/parser/BufferedStringTest.java | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/h2o-core/src/test/java/water/parser/BufferedStringTest.java b/h2o-core/src/test/java/water/parser/BufferedStringTest.java index 107d329b630..b99b191d537 100644 --- a/h2o-core/src/test/java/water/parser/BufferedStringTest.java +++ b/h2o-core/src/test/java/water/parser/BufferedStringTest.java @@ -30,6 +30,7 @@ public void testWrite_impl() throws Exception { assertArrayEquals(expected, actual); } + @Ignore("This test fails currently - bugs in AutoBuffer, probably") @Test public void testRead_impl() throws Exception { final String source = "this is not a string"; @@ -37,9 +38,9 @@ public void testRead_impl() throws Exception { AutoBuffer ab = new AutoBuffer(); sut1.write_impl(ab); ab.bufClose(); -// BufferedString sut2 = new BufferedString("what?"); -// sut2.read_impl(ab); -// assertEquals(sut1, sut2); + BufferedString sut2 = new BufferedString("what?"); + sut2.read_impl(ab); + assertEquals(sut1, sut2); } @Test @@ -50,7 +51,7 @@ public void testCompareTo() throws Exception { assertEquals(2, sut1.compareTo(new BufferedString("this is not a stri"))); } - @Test + @Test @Ignore // this is a stub public void testHashCode() throws Exception { } @@ -72,51 +73,52 @@ public void testAddChar() throws Exception { // assertArrayEquals(source.getBytes(), bytes); } - @Test + @Test @Ignore // this is a stub public void testAddBuff() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testToString() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testBytesToString() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testToString1() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testToBufferedString() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testSet() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testSet1() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testSet2() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testSetOff() throws Exception { } + @SuppressWarnings("EqualsBetweenInconvertibleTypes") @Test public void testEquals() throws Exception { BufferedString sut = new BufferedString("abc"); @@ -180,17 +182,17 @@ public void testGetBuffer() throws Exception { expected, actual); } - @Test + @Test @Ignore // this is a stub public void testGetOffset() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testLength() throws Exception { } - @Test + @Test @Ignore // this is a stub public void testGetNumericType() throws Exception { } From f0c13a2adad97c19f6331a53cea601d25aae20eb Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Tue, 7 Mar 2017 17:31:20 -0800 Subject: [PATCH 6/9] Blocking OOM test as flaky and testing unnecessary feature. --- h2o-core/src/test/java/water/OOMTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/h2o-core/src/test/java/water/OOMTest.java b/h2o-core/src/test/java/water/OOMTest.java index 86331df99fd..5e5f5501c50 100644 --- a/h2o-core/src/test/java/water/OOMTest.java +++ b/h2o-core/src/test/java/water/OOMTest.java @@ -127,7 +127,8 @@ public void testParseMemoryStress() { public static void main(String[] args) { stall_till_cloudsize(args, 1); try { - new OOMTest().testParseMemoryStress(); // Throws on assertion error + System.out.println("All disabled."); + //new OOMTest().testParseMemoryStress(); // Throws on assertion error } catch( Throwable e ) { Log.err(e); StringWriter sw = new StringWriter(); From 2adeb091f03510b83787e1492612ebbcf6c81161 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Tue, 7 Mar 2017 21:17:30 -0800 Subject: [PATCH 7/9] Removed commented-out asserts, on Michal's request. --- h2o-core/src/main/java/water/H2O.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/h2o-core/src/main/java/water/H2O.java b/h2o-core/src/main/java/water/H2O.java index 2dfea80ac12..cb811b6d8cc 100644 --- a/h2o-core/src/main/java/water/H2O.java +++ b/h2o-core/src/main/java/water/H2O.java @@ -1536,8 +1536,6 @@ static boolean larger( int nnn, int old ) { public final int size() { return _memary.length; } final H2ONode leader() { -// assert _memary != null; -// assert _memary.length > 0; return _memary[0]; } From a0d5ac8cc6f77c776395eb34a5fcccf2f7381624 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Wed, 8 Mar 2017 11:55:38 -0800 Subject: [PATCH 8/9] Added more error reporting in case of AWS failures. S3 test was consistently failing today. --- h2o-persist-s3/src/main/java/water/persist/PersistS3.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/h2o-persist-s3/src/main/java/water/persist/PersistS3.java b/h2o-persist-s3/src/main/java/water/persist/PersistS3.java index 0f92a5a3fbe..9a2d7b5a95c 100644 --- a/h2o-persist-s3/src/main/java/water/persist/PersistS3.java +++ b/h2o-persist-s3/src/main/java/water/persist/PersistS3.java @@ -82,7 +82,12 @@ public H2OAWSCredentialsProviderChain() { try { return new PropertiesCredentials(credentials); } catch (IOException e) { - throw new AmazonClientException("Unable to load AWS credentials from file " + credentials); + Log.debug( + "Unable to load AWS credentials from file " + credentials + + "; exists? " + credentials.exists() + ", canRead? " + credentials.canRead() + + ", size=" + credentials.length() + "; problem: " + e.getMessage()); + throw new AmazonClientException( + "PersistS3. Unable to load AWS credentials from file " + credentials + ": " + e.getMessage()); } } @@ -365,6 +370,7 @@ public Key uriToKey(URI uri) throws IOException { if (e.getErrorCode().contains("404")) { throw new IOException(e); } else { + Log.err("AWS failed for " + Arrays.toString(parts) + ": " + e.getMessage()); throw e; } } From 3831db9865884da7485bd16afa295c3fd80434d6 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Thu, 9 Mar 2017 11:07:39 -0800 Subject: [PATCH 9/9] Restored OOMTest, for a moment --- h2o-core/src/test/java/water/OOMTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/h2o-core/src/test/java/water/OOMTest.java b/h2o-core/src/test/java/water/OOMTest.java index 5e5f5501c50..87288625246 100644 --- a/h2o-core/src/test/java/water/OOMTest.java +++ b/h2o-core/src/test/java/water/OOMTest.java @@ -127,8 +127,8 @@ public void testParseMemoryStress() { public static void main(String[] args) { stall_till_cloudsize(args, 1); try { - System.out.println("All disabled."); - //new OOMTest().testParseMemoryStress(); // Throws on assertion error + // TODO(vlad): disable it + new OOMTest().testParseMemoryStress(); // Throws on assertion error } catch( Throwable e ) { Log.err(e); StringWriter sw = new StringWriter();