counts = new WordCountTask().doAll(fr.vec(0))._counts;
diff --git a/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java b/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java
index 10a5c74f6bb..2e0704ae9be 100644
--- a/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java
+++ b/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java
@@ -7,11 +7,11 @@
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
+import static water.util.FileUtils.*;
import java.io.*;
import java.util.concurrent.TimeUnit;
-import static water.TestUtil.find_test_file_static;
import static water.mojo.glm.GlmMojoBenchHelper.*;
/**
@@ -41,7 +41,7 @@
@Setup
public void setup() throws IOException {
- File f = find_test_file_static("smalldata/airlines/allyears2k.zip");
+ File f = getFile("smalldata/airlines/allyears2k.zip");
mojo = (GlmMojoModel) loadMojo("airlines");
diff --git a/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java b/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java
index f6a43f5dcf5..f6f01984111 100644
--- a/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java
+++ b/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java
@@ -7,12 +7,13 @@
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
+import water.util.FileUtils;
import java.io.*;
import java.util.concurrent.TimeUnit;
-import static water.TestUtil.find_test_file_static;
import static water.mojo.glm.GlmMojoBenchHelper.*;
+import static water.util.FileUtils.*;
/**
* GLM MOJO scoring benchmark (multinomial)
@@ -41,7 +42,7 @@
@Setup
public void setup() throws IOException {
- File f = find_test_file_static("smalldata/flow_examples/mnist/test.csv.gz");
+ File f = getFile("smalldata/flow_examples/mnist/test.csv.gz");
mojo = (GlmMultinomialMojoModel) loadMojo("mnist");
diff --git a/h2o-core/src/main/java/water/fvec/NFSFileVec.java b/h2o-core/src/main/java/water/fvec/NFSFileVec.java
index ed1f0a0d20a..078ed0e8b46 100644
--- a/h2o-core/src/main/java/water/fvec/NFSFileVec.java
+++ b/h2o-core/src/main/java/water/fvec/NFSFileVec.java
@@ -1,9 +1,11 @@
package water.fvec;
import java.io.File;
+import java.io.IOException;
import water.*;
import water.persist.PersistNFS;
+import water.util.FileUtils;
/** A NFS distributed file-backed Vector
*
@@ -29,6 +31,11 @@ public static NFSFileVec make(File f) {
return nfs;
}
+ public static NFSFileVec make(String fname) throws IOException {
+ File f = FileUtils.getFile(fname);
+ return NFSFileVec.make(f);
+ }
+
/** Make a new NFSFileVec key which holds the filename implicitly. This name
* is used by the Chunks to load data on-demand.
* @return A NFSFileVec mapped to this file. */
diff --git a/h2o-core/src/main/java/water/util/FileUtils.java b/h2o-core/src/main/java/water/util/FileUtils.java
index 91886bdb7c4..809b338a9c0 100644
--- a/h2o-core/src/main/java/water/util/FileUtils.java
+++ b/h2o-core/src/main/java/water/util/FileUtils.java
@@ -2,6 +2,7 @@
import org.eclipse.jetty.io.EofException;
import water.Key;
+import water.fvec.NFSFileVec;
import java.io.*;
import java.net.URI;
@@ -64,6 +65,57 @@ else if (file.isDirectory()) {
return false;
}
+
+ /** Hunt for files in likely places. Null if cannot find.
+ * @param fname filename
+ * @return Found file or null */
+ public static File locateFile(String fname) {
+ // When run from eclipse, the working directory is different.
+ // Try pointing at another likely place
+ File file = new File(fname);
+ if( !file.exists() )
+ file = new File("target/" + fname);
+ if( !file.exists() )
+ file = new File("../" + fname);
+ if( !file.exists() )
+ file = new File("../../" + fname);
+ if( !file.exists() )
+ file = new File("../target/" + fname);
+ if( !file.exists() )
+ file = new File(StringUtils.expandPath(fname));
+ if( !file.exists() )
+ file = null;
+ return file;
+ }
+
+ private static void check(boolean cond, String msg) throws IOException{
+ if (!cond) throw new IOException(msg);
+ }
+
+ private static void checkFileEntry(String name, File file) throws IOException {
+ check(file != null, "File not found: " + name);
+ check(file.exists(), "File should exist: " + name);
+ }
+
+ public static void checkFile(File file, String name) throws IOException {
+ checkFileEntry(name, file);
+ check(file.isFile(), "Expected a file: " + name);
+ check(file.canRead(), "Expected a readable file: " + name);
+ }
+
+ public static File getFile(String fname) throws IOException {
+ File f = locateFile(fname);
+ checkFile(f, fname);
+ return f;
+ }
+
+ public static File[] contentsOf(File folder, String name) throws IOException {
+ checkFileEntry(name, folder);
+ if (!folder.isDirectory()) throw new IOException("Expected a folder: " + name);
+ File[] files = folder.listFiles();
+ if (files == null) throw new IOException("Cannot read folder: " + folder);
+ return files;
+ }
/** Transform given key to a string which can be used as a file name. */
public static String keyToFileName(Key k) {
diff --git a/h2o-core/src/test/java/hex/ConfusionMatrixTest.java b/h2o-core/src/test/java/hex/ConfusionMatrixTest.java
index 2c7eb3803af..6be71d559c5 100644
--- a/h2o-core/src/test/java/hex/ConfusionMatrixTest.java
+++ b/h2o-core/src/test/java/hex/ConfusionMatrixTest.java
@@ -8,6 +8,7 @@
import water.TestUtil;
import water.fvec.Frame;
import water.util.ArrayUtils;
+import water.util.FileUtils;
import water.util.VecUtils;
import static water.util.FrameUtils.parseFrame;
@@ -196,8 +197,8 @@ public void testDifferentDomainsNumericVectors() {
private void simpleCMTest(String f1, String f2, String[] expectedActualDomain, String[] expectedPredictDomain, String[] expectedDomain, double[][] expectedCM, boolean debug) {
try {
- Frame v1 = parseFrame(Key.make("v1.hex"), find_test_file(f1));
- Frame v2 = parseFrame(Key.make("v2.hex"), find_test_file(f2));
+ Frame v1 = parseFrame(Key.make("v1.hex"), FileUtils.getFile(f1));
+ Frame v2 = parseFrame(Key.make("v2.hex"), FileUtils.getFile(f2));
if (!v1.isCompatible(v2)) {
Frame old = null;
v2 = new Frame(v1.makeCompatible(old = v2));
diff --git a/h2o-core/src/test/java/water/KVTest.java b/h2o-core/src/test/java/water/KVTest.java
index 8f57d2e5710..4624faf9c22 100644
--- a/h2o-core/src/test/java/water/KVTest.java
+++ b/h2o-core/src/test/java/water/KVTest.java
@@ -5,8 +5,8 @@
import java.io.File;
import water.fvec.Chunk;
-import water.fvec.Frame;
import water.fvec.NFSFileVec;
+import water.util.FileUtils;
import water.util.UnsafeUtils;
public class KVTest extends TestUtil {
@@ -126,7 +126,7 @@
long start = System.currentTimeMillis();
NFSFileVec nfs = null;
try {
- File file = find_test_file("build/h2o-core.jar");
+ File file = FileUtils.locateFile("build/h2o-core.jar");
if( file == null ) return; // Nothing to test
// Return a Key mapping to a NFSFileVec over the file
nfs = NFSFileVec.make(file);
diff --git a/h2o-core/src/test/java/water/OOMTest.java b/h2o-core/src/test/java/water/OOMTest.java
index 59436d2b8e5..86331df99fd 100644
--- a/h2o-core/src/test/java/water/OOMTest.java
+++ b/h2o-core/src/test/java/water/OOMTest.java
@@ -7,6 +7,7 @@
import org.junit.*;
import water.fvec.Frame;
import water.fvec.Vec;
+import static water.util.FileUtils.*;
import water.util.Log;
@Ignore
@@ -93,8 +94,8 @@
@Test @Ignore
public void testParseMemoryStress() {
// "bigdata directory is not always available"
- if( find_test_file_static("bigdata/laptop/usecases/cup98LRN_z.csv") == null ) return;
- if( find_test_file_static("bigdata/laptop/usecases/cup98VAL_z.csv") == null ) return;
+ if( locateFile("bigdata/laptop/usecases/cup98LRN_z.csv") == null ) return;
+ if( locateFile("bigdata/laptop/usecases/cup98VAL_z.csv") == null ) return;
ArrayList frames = new ArrayList<>();
File ice = new File(water.H2O.ICE_ROOT.toString(),"ice" + water.H2O.API_PORT);
String[] dirs = ice.list();
diff --git a/h2o-core/src/test/java/water/TestUtil.java b/h2o-core/src/test/java/water/TestUtil.java
index 838601644e7..7ef597f581e 100644
--- a/h2o-core/src/test/java/water/TestUtil.java
+++ b/h2o-core/src/test/java/water/TestUtil.java
@@ -1,6 +1,5 @@
package water;
-import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Ignore;
@@ -13,10 +12,8 @@
import water.parser.DefaultParserProviders;
import water.parser.ParseDataset;
import water.parser.ParseSetup;
-import water.util.Log;
-import water.util.StringUtils;
+import water.util.*;
import water.util.Timer;
-import water.util.TwoDimTable;
import java.io.File;
import java.io.IOException;
@@ -24,9 +21,7 @@
import java.lang.reflect.Method;
import java.util.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
@Ignore("Support for tests, but no actual tests here")
public class TestUtil extends Iced {
@@ -192,28 +187,6 @@ public void evaluate() throws Throwable {}
// ==== Data Frame Creation Utilities ====
- /** Hunt for test files in likely places. Null if cannot find.
- * @param fname Test filename
- * @return Found file or null */
- public static File find_test_file_static(String fname) {
- // When run from eclipse, the working directory is different.
- // Try pointing at another likely place
- File file = new File(fname);
- if( !file.exists() )
- file = new File("target/" + fname);
- if( !file.exists() )
- file = new File("../" + fname);
- if( !file.exists() )
- file = new File("../../" + fname);
- if( !file.exists() )
- file = new File("../target/" + fname);
- if( !file.exists() )
- file = new File(StringUtils.expandPath(fname));
- if( !file.exists() )
- file = null;
- return file;
- }
-
/** Compare 2 frames
* @param fr1 Frame
* @param fr2 Frame
@@ -238,53 +211,43 @@ public static boolean isBitIdentical(Frame fr1, Frame fr2) {
return isIdenticalUpToRelTolerance(fr1,fr2,0);
}
- /** Hunt for test files in likely places. Null if cannot find.
- * @param fname Test filename
- * @return Found file or null */
- protected File find_test_file(String fname) {
- return find_test_file_static(fname);
- }
-
- private static void checkFileEntry(String name, File file) {
- assertNotNull("File not found: " + name, file);
- assertTrue("File should exist: " + name, file.exists());
- }
-
- public static void checkFile(String name, File file) {
- checkFileEntry(name, file);
- assertTrue("Expected a readable file: " + name, file.canRead());
- }
-
- static File[] checkFolder(String name, File folder) {
- checkFileEntry(name, folder);
- assertTrue("Expected a folder: " + name, folder.isDirectory());
- File[] files = folder.listFiles();
- assertNotNull("No files found in " + folder, files);
- return files;
+ static File[] contentsOf(String name, File folder) {
+ try {
+ return FileUtils.contentsOf(folder, name);
+ } catch (IOException ioe) {
+ fail(ioe.getMessage());
+ return null;
+ }
}
/** Find & parse a CSV file. NPE if file not found.
* @param fname Test filename
* @return Frame or NPE */
- public static Frame parse_test_file( String fname ) { return parse_test_file(Key.make(),fname); }
+ public static Frame parse_test_file( String fname ) {
+ return parse_test_file(Key.make(),fname);
+ }
+
+ public static NFSFileVec makeNfsFileVec(String fname) {
+ try {
+ return NFSFileVec.make(fname);
+ } catch (IOException ioe) {
+ fail(ioe.getMessage());
+ return null;
+ }
+ }
+
public static Frame parse_test_file( Key outputKey, String fname) {
- File f = find_test_file_static(fname);
- checkFile(fname, f);
- NFSFileVec nfs = NFSFileVec.make(f);
+ NFSFileVec nfs = makeNfsFileVec(fname);
return ParseDataset.parse(outputKey, nfs._key);
}
- protected Frame parse_test_file( Key outputKey, String fname , boolean guessSetup) {
- File f = find_test_file(fname);
- checkFile(fname, f);
- NFSFileVec nfs = NFSFileVec.make(f);
+ protected Frame parse_test_file( Key outputKey, String fname, boolean guessSetup) {
+ NFSFileVec nfs = makeNfsFileVec(fname);
return ParseDataset.parse(outputKey, new Key[]{nfs._key}, true, ParseSetup.guessSetup(new Key[]{nfs._key},false,1));
}
protected Frame parse_test_file( String fname, String na_string, int check_header, byte[] column_types ) {
- File f = find_test_file_static(fname);
- checkFile(fname, f);
- NFSFileVec nfs = NFSFileVec.make(f);
+ NFSFileVec nfs = makeNfsFileVec(fname);
Key[] res = {nfs._key};
@@ -317,8 +280,8 @@ protected Frame parse_test_file( String fname, String na_string, int check_heade
* @param fname Test filename
* @return Frame or NPE */
protected Frame parse_test_folder( String fname ) {
- File folder = find_test_file(fname);
- File[] files = checkFolder(fname, folder);
+ File folder = FileUtils.locateFile(fname);
+ File[] files = contentsOf(fname, folder);
Arrays.sort(files);
ArrayList keys = new ArrayList<>();
for( File f : files )
@@ -338,8 +301,8 @@ protected Frame parse_test_folder( String fname ) {
* @return
*/
protected static Frame parse_test_folder( String fname, String na_string, int check_header, byte[] column_types ) {
- File folder = find_test_file_static(fname);
- File[] files = checkFolder(fname, folder);
+ File folder = FileUtils.locateFile(fname);
+ File[] files = contentsOf(fname, folder);
Arrays.sort(files);
ArrayList keys = new ArrayList<>();
for( File f : files )
@@ -756,7 +719,7 @@ public Frame prepare() {
public void done(Frame frame) {
if (generatedFile != null) {
generatedFile.deleteOnExit();
- FileUtils.deleteQuietly(generatedFile);
+ org.apache.commons.io.FileUtils.deleteQuietly(generatedFile);
}
}
}
diff --git a/h2o-core/src/test/java/water/fvec/FVecTest.java b/h2o-core/src/test/java/water/fvec/FVecTest.java
index ab4dfff4ab8..9b5cd516194 100644
--- a/h2o-core/src/test/java/water/fvec/FVecTest.java
+++ b/h2o-core/src/test/java/water/fvec/FVecTest.java
@@ -5,6 +5,7 @@
import org.junit.Test;
import water.*;
import water.util.ArrayUtils;
+import water.util.FileUtils;
import java.io.File;
@@ -47,7 +48,7 @@ public static Key makeByteVec(Key k, String... data) {
// ==========================================================================
@Test public void testBasicCRUD() {
// Make and insert a FileVec to the global store
- File file = find_test_file("./smalldata/junit/cars.csv");
+ File file = FileUtils.locateFile("./smalldata/junit/cars.csv");
NFSFileVec nfs = NFSFileVec.make(file);
int sum = ArrayUtils.sum(new ByteHisto().doAll(nfs)._x);
assertEquals(file.length(),sum);
@@ -107,8 +108,7 @@ public static Key makeByteVec(Key k, String... data) {
// Test making a appendable vector from a plain vector
@Test public void testNewVec() {
// Make and insert a File8Vec to the global store
- File file = find_test_file("./smalldata/junit/cars.csv");
- NFSFileVec nfs = NFSFileVec.make(file);
+ NFSFileVec nfs = TestUtil.makeNfsFileVec("./smalldata/junit/cars.csv");
Vec res = new TestNewVec().doAll(new byte[]{Vec.T_NUM},nfs).outputFrame(new String[]{"v"},new String[][]{null}).anyVec();
assertEquals(nfs.at8(0)+1,res.at8(0));
assertEquals(nfs.at8(1)+1,res.at8(1));
diff --git a/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java b/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java
index ee01aec7cbf..a03af0a021c 100644
--- a/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java
+++ b/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java
@@ -5,6 +5,7 @@
import water.*;
import water.parser.ParseDataset;
+import water.util.FileUtils;
import water.util.FrameUtils;
import water.util.Log;
@@ -12,8 +13,8 @@
@BeforeClass public static void setup() { stall_till_cloudsize(1); }
@Test public void testProstate(){
NFSFileVec[] nfs = new NFSFileVec[]{
- NFSFileVec.make(find_test_file("smalldata/logreg/prostate.csv")),
- NFSFileVec.make(find_test_file("smalldata/covtype/covtype.20k.data"))};
+ TestUtil.makeNfsFileVec("smalldata/logreg/prostate.csv"),
+ TestUtil.makeNfsFileVec("smalldata/covtype/covtype.20k.data")};
//NFSFileVec.make(find_test_file("bigdata/laptop/usecases/cup98VAL_z.csv"))};
for (NFSFileVec fv : nfs) {
Frame fr = ParseDataset.parse(Key.make(), fv._key);
diff --git a/h2o-core/src/test/java/water/fvec/WordCountBigTest.java b/h2o-core/src/test/java/water/fvec/WordCountBigTest.java
index 27fc231bf0a..02fabe1927d 100644
--- a/h2o-core/src/test/java/water/fvec/WordCountBigTest.java
+++ b/h2o-core/src/test/java/water/fvec/WordCountBigTest.java
@@ -1,15 +1,15 @@
package water.fvec;
import org.junit.*;
+import water.util.FileUtils;
import java.io.*;
public class WordCountBigTest extends WordCountTest {
@Test public void testWordCountWiki() throws IOException {
String best = "/home/0xdiag/datasets/wiki.xml";
- File file = find_test_file(best);
- if( file==null ) file = find_test_file("../datasets/Wiki_20130805.xml");
- if( file==null ) throw new FileNotFoundException(best);
+ File file = FileUtils.locateFile(best);
+ if( file==null ) file = FileUtils.getFile("../datasets/Wiki_20130805.xml");
doWordCount(file);
}
diff --git a/h2o-core/src/test/java/water/fvec/WordCountTest.java b/h2o-core/src/test/java/water/fvec/WordCountTest.java
index d33da7eb9ea..74ccffbe658 100644
--- a/h2o-core/src/test/java/water/fvec/WordCountTest.java
+++ b/h2o-core/src/test/java/water/fvec/WordCountTest.java
@@ -8,13 +8,14 @@
import java.util.concurrent.atomic.AtomicLong;
import water.*;
import water.nbhm.NonBlockingHashMap;
+import water.util.FileUtils;
public class WordCountTest extends TestUtil {
@BeforeClass static public void setup() { stall_till_cloudsize(1); }
// ==========================================================================
@Test public void testWordCount() throws IOException {
- File file = find_test_file("./smalldata/junit/cars.csv");
+ File file = FileUtils.getFile("./smalldata/junit/cars.csv");
doWordCount(file);
}
diff --git a/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java b/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java
index 8a66e195467..75f39a5c6fe 100644
--- a/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java
+++ b/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java
@@ -1,7 +1,7 @@
package water.network;
import org.junit.Test;
-import water.TestUtil;
+import water.util.FileUtils;
import javax.net.ssl.SSLException;
import java.io.IOException;
@@ -14,6 +14,7 @@
import java.util.concurrent.CyclicBarrier;
import static org.junit.Assert.*;
+import static water.util.FileUtils.*;
public class SSLSocketChannelFactoryTest {
@@ -23,9 +24,9 @@
public void shouldHandshake() throws IOException, SSLContextException, BrokenBarrierException, InterruptedException {
SSLProperties props = new SSLProperties();
props.put("h2o_ssl_protocol", SecurityUtils.defaultTLSVersion());
- props.put("h2o_ssl_jks_internal", TestUtil.find_test_file_static("src/test/resources/keystore.jks").getPath());
+ props.put("h2o_ssl_jks_internal", getFile("src/test/resources/keystore.jks").getPath());
props.put("h2o_ssl_jks_password", "password");
- props.put("h2o_ssl_jts", TestUtil.find_test_file_static("src/test/resources/cacerts.jks").getPath());
+ props.put("h2o_ssl_jts", getFile("src/test/resources/cacerts.jks").getPath());
props.put("h2o_ssl_jts_password", "password");
final SSLSocketChannelFactory factory = new SSLSocketChannelFactory(props);
diff --git a/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java b/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java
index 9cbaf2a1725..725dd86e06f 100644
--- a/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java
+++ b/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java
@@ -10,6 +10,7 @@
import water.fvec.Vec;
import water.fvec.Frame;
import water.fvec.NFSFileVec;
+import water.util.FileUtils;
public class ParseCompressedAndXLSTest extends TestUtil {
@BeforeClass static public void setup() { stall_till_cloudsize(5); }
@@ -50,8 +51,7 @@
@Test public void testXLSBadArgs(){
Frame k1 = null;
try {
- File f = find_test_file("smalldata/airlines/AirlinesTest.csv.zip");
- NFSFileVec nfs = NFSFileVec.make(f);
+ NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/airlines/AirlinesTest.csv.zip");
byte[] ctypes = new byte[12];
for(int i=0; i < 12; i++) ctypes[i] = Vec.T_NUM;
ParseSetup setup = new ParseSetup(XLS_INFO,
diff --git a/h2o-core/src/test/java/water/parser/ParseFolderTest.java b/h2o-core/src/test/java/water/parser/ParseFolderTest.java
index 8d25b2ba621..e903693992b 100644
--- a/h2o-core/src/test/java/water/parser/ParseFolderTest.java
+++ b/h2o-core/src/test/java/water/parser/ParseFolderTest.java
@@ -7,6 +7,7 @@
import water.TestUtil;
import water.fvec.Frame;
import water.fvec.NFSFileVec;
+import water.util.FileUtils;
public class ParseFolderTest extends TestUtil {
@BeforeClass static public void setup() { stall_till_cloudsize(5); }
@@ -24,7 +25,7 @@
}
@Test public void testSameFile() {
- File f = find_test_file("smalldata/iris/iris_wheader.csv");
+ File f = FileUtils.locateFile("smalldata/iris/iris_wheader.csv");
NFSFileVec nfs1 = NFSFileVec.make(f);
NFSFileVec nfs2 = NFSFileVec.make(f);
Frame fr = null;
diff --git a/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java b/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java
index 4ca24a06e5f..41cc3b154ae 100644
--- a/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java
+++ b/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java
@@ -1,6 +1,7 @@
package water.parser;
import java.io.File;
+import java.io.IOException;
import java.util.Arrays;
import org.junit.*;
import water.Job;
@@ -9,6 +10,7 @@
import water.fvec.Frame;
import water.fvec.NFSFileVec;
import water.fvec.Vec;
+import water.util.FileUtils;
public class ParseFolderTestBig extends TestUtil {
@BeforeClass static public void setup() { stall_till_cloudsize(1); }
@@ -53,18 +55,20 @@ public void testBIGSVM() {
String fname = "bigdata/cust_K/1m.svm";
Frame k1 = null;
try {
- File f = find_test_file(fname);
- assert f != null && f.exists():" file not found: " + fname;
+ File f = FileUtils.getFile(fname);
NFSFileVec nfs = NFSFileVec.make(f);
- Job job = ParseDataset.parse(Key.make("BIGSVM.hex"),new Key[]{nfs._key},true,ParseSetup.guessSetup(new Key[]{nfs._key}, false, ParseSetup.GUESS_HEADER),false)._job;
- while( job.progress() < 1.0 ) {
- System.out.print(((int)(job.progress()*1000.0))/10.0 + "% ");
- try { Thread.sleep(1000); } catch (InterruptedException ignore) { /*comment to disable ideaJ warning*/}
+ Job job = ParseDataset.parse(Key.make("BIGSVM.hex"), new Key[]{nfs._key}, true, ParseSetup.guessSetup(new Key[]{nfs._key}, false, ParseSetup.GUESS_HEADER), false)._job;
+ while (job.progress() < 1.0) {
+ System.out.print(((int) (job.progress() * 1000.0)) / 10.0 + "% ");
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignore) { /*comment to disable ideaJ warning*/}
}
System.out.println();
k1 = job.get();
System.out.println(k1.toString());
-
+ } catch (IOException ioe) {
+ Assert.fail("File not found: " + fname + " - " + ioe.getMessage());
} finally {
if( k1 != null ) k1.delete();
}
diff --git a/h2o-core/src/test/java/water/parser/ParseProgressTest.java b/h2o-core/src/test/java/water/parser/ParseProgressTest.java
index a0b49c47dde..61a3d4aefa6 100644
--- a/h2o-core/src/test/java/water/parser/ParseProgressTest.java
+++ b/h2o-core/src/test/java/water/parser/ParseProgressTest.java
@@ -7,6 +7,7 @@
import water.TestUtil;
import water.fvec.Frame;
import water.util.FileIntegrityChecker;
+import water.util.FileUtils;
public class ParseProgressTest extends TestUtil {
// Attempt a multi-jvm parse of covtype.
@@ -15,7 +16,7 @@
String[] covtype_locations = new String[]{"../datasets/UCI/UCI-large/covtype/covtype.data", "../../datasets/UCI/UCI-large/covtype/covtype.data", "../datasets/UCI/UCI-large/covtype/covtype.data.gz", "../demo/UCI-large/covtype/covtype.data", };
File f = null;
for( String covtype_location : covtype_locations ) {
- f = find_test_file( covtype_location );
+ f = FileUtils.locateFile(covtype_location);
if( f != null && f.exists() )
break;
}
diff --git a/h2o-core/src/test/java/water/parser/ParserTest.java b/h2o-core/src/test/java/water/parser/ParserTest.java
index e8122ae0e7c..bdf2cdd296c 100644
--- a/h2o-core/src/test/java/water/parser/ParserTest.java
+++ b/h2o-core/src/test/java/water/parser/ParserTest.java
@@ -9,9 +9,9 @@
import water.*;
import water.api.schemas3.ParseSetupV3;
import water.fvec.*;
+import water.util.FileUtils;
import water.util.Log;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
@@ -845,7 +845,7 @@ static void testParsed(Frame fr, double[][] expected, int len) {
}) {
try {
Log.info("Trying to parse " + f);
- NFSFileVec nfs = NFSFileVec.make(find_test_file(f));
+ NFSFileVec nfs = TestUtil.makeNfsFileVec(f);
Frame fr = ParseDataset.parse(Key.make(), new Key[]{nfs._key}, delete_on_done, true /*single quote*/, check_header);
fr.delete();
} catch (Throwable t) {
@@ -886,7 +886,7 @@ public void testPubDev2897() {
@Test
public void testParserRespectsSpecifiedColNum() {
- Vec fv = NFSFileVec.make(find_test_file("smalldata/jira/runit_pubdev_3590_unexpected_column.csv"));
+ Vec fv = TestUtil.makeNfsFileVec("smalldata/jira/runit_pubdev_3590_unexpected_column.csv");
Key fkey = Key.make("data4cols");
try {
Key[] keys = new Key[]{fv._key};
diff --git a/h2o-core/src/test/java/water/parser/ParserTestARFF.java b/h2o-core/src/test/java/water/parser/ParserTestARFF.java
index 7b011baba04..8e4cbe040dc 100644
--- a/h2o-core/src/test/java/water/parser/ParserTestARFF.java
+++ b/h2o-core/src/test/java/water/parser/ParserTestARFF.java
@@ -11,6 +11,8 @@
import water.fvec.Frame;
import water.fvec.NFSFileVec;
import water.fvec.Vec;
+import water.util.FileUtils;
+
import static water.parser.ParserTest.makeByteVec;
import java.util.Arrays;
@@ -91,7 +93,7 @@ private void testColNames(String[] dataset, String[] exp, int len, String sep) {
* @param fname Test filename
* @return Frame or NPE */
private Frame parse_test_file_single_quotes( String fname ) {
- NFSFileVec nfs = NFSFileVec.make(find_test_file(fname));
+ NFSFileVec nfs = TestUtil.makeNfsFileVec(fname);
return ParseDataset.parse(Key.make(), new Key[]{nfs._key}, true, true /*single quote*/, ParseSetup.GUESS_HEADER);
}
diff --git a/h2o-core/src/test/java/water/rapids/MungingTest.java b/h2o-core/src/test/java/water/rapids/MungingTest.java
index e48cb8dc901..0ceea45aaf5 100644
--- a/h2o-core/src/test/java/water/rapids/MungingTest.java
+++ b/h2o-core/src/test/java/water/rapids/MungingTest.java
@@ -6,7 +6,7 @@
import water.*;
import water.fvec.*;
import water.parser.ParseDataset;
-import water.util.Log;
+import water.util.FileUtils;
import java.io.InputStream;
import java.io.OutputStream;
@@ -32,7 +32,7 @@ private void copyStream(OutputStream os, InputStream is, final int buffer_size)
}
@Ignore @Test public void run2() throws Exception {
System.out.println("Running run2 ...");
- NFSFileVec nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/step1.csv"));
+ NFSFileVec nfs = TestUtil.makeNfsFileVec("/home/mdowle/devtestdata/step1.csv");
Frame frame = ParseDataset.parse(Key.make(), nfs._key); // look into parse() to manip column types
System.out.println("Loaded file, now calling Query ...");
// new RadixOrder(frame, true, new int[] {0,1}); // group by 0=id, 1=date and sum 3 == quantity
@@ -43,12 +43,12 @@ private void copyStream(OutputStream os, InputStream is, final int buffer_size)
@Ignore @Test public void run3() throws Exception {
System.out.println("Running run3 ...");
- NFSFileVec nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/step1_subset.csv"));
+ NFSFileVec nfs = TestUtil.makeNfsFileVec("/home/mdowle/devtestdata/step1_subset.csv");
//NFSFileVec nfs = NFSFileVec.make(find_test_file("/users/arno/devtestdata/step1_subset.csv"));
Frame leftFrame = ParseDataset.parse(Key.make(), nfs._key);
//nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/fullsize.csv"));
- nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/fullsize.csv"));
+ nfs = NFSFileVec.make(FileUtils.locateFile("/home/mdowle/devtestdata/fullsize.csv"));
//nfs = NFSFileVec.make(find_test_file("/users/arno/devtestdata/fullsize.csv"));
Frame rightFrame = ParseDataset.parse(Key.make(), nfs._key); // look into parse() to manip column types
diff --git a/h2o-core/src/test/java/water/rapids/RapidsTest.java b/h2o-core/src/test/java/water/rapids/RapidsTest.java
index 5185a6c52eb..c167cdfcee4 100644
--- a/h2o-core/src/test/java/water/rapids/RapidsTest.java
+++ b/h2o-core/src/test/java/water/rapids/RapidsTest.java
@@ -14,6 +14,7 @@
import water.rapids.ast.params.AstStr;
import water.rapids.vals.ValFrame;
import water.util.ArrayUtils;
+import water.util.FileUtils;
import water.util.Log;
import java.io.File;
@@ -461,7 +462,7 @@ static boolean checkSaneFrame_impl() {
parse_test_file(Key.make("weather.hex"),"smalldata/chicago/chicagoAllWeather.csv");
parse_test_file(Key.make( "crimes.hex"),"smalldata/chicago/chicagoCrimes10k.csv.zip");
String fname = "smalldata/chicago/chicagoCensus.csv";
- File f = find_test_file(fname);
+ File f = FileUtils.locateFile(fname);
assert f != null && f.exists():" file not found: " + fname;
NFSFileVec nfs = NFSFileVec.make(f);
ParseSetup ps = ParseSetup.guessSetup(new Key[]{nfs._key}, false, 1);
diff --git a/h2o-core/src/test/java/water/udf/UdfTest.java b/h2o-core/src/test/java/water/udf/UdfTest.java
index b1fed577767..0085f1389b9 100644
--- a/h2o-core/src/test/java/water/udf/UdfTest.java
+++ b/h2o-core/src/test/java/water/udf/UdfTest.java
@@ -17,6 +17,7 @@
import static water.udf.specialized.Dates.*;
import static water.udf.specialized.Doubles.*;
import static water.udf.specialized.Strings.*;
+import static water.util.FileUtils.*;
/**
* Test for UDF
*/
diff --git a/h2o-core/src/test/java/water/udf/UdfTestBase.java b/h2o-core/src/test/java/water/udf/UdfTestBase.java
index df8eea3da8a..20bdd60a38c 100644
--- a/h2o-core/src/test/java/water/udf/UdfTestBase.java
+++ b/h2o-core/src/test/java/water/udf/UdfTestBase.java
@@ -4,16 +4,15 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import water.H2OStarter;
-import water.Iced;
import water.Scope;
import water.TestUtil;
import water.fvec.NFSFileVec;
import water.fvec.Vec;
+import water.util.FileUtils;
import java.io.File;
+import java.io.IOException;
import java.lang.reflect.Method;
-import static water.TestUtil.*;
/**
* All test functionality specific for udf (not actually),
@@ -52,17 +51,11 @@ public void bye() {
}
- public static Vec loadFile(String fname) {
- File f = getFile(fname);
+ public static Vec loadFile(String fname) throws IOException {
+ File f = FileUtils.getFile(fname);
return NFSFileVec.make(f);
}
- public static File getFile(String fname) {
- File f = find_test_file_static(fname);
- checkFile(fname, f);
- return f;
- }
-
// the following code exists or else gradlew will complain; also, it checks assertions
@Test
public void testAssertionsEnabled() throws Exception {
diff --git a/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java b/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java
index 50fbbe0d826..610fe5cc424 100644
--- a/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java
+++ b/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java
@@ -10,10 +10,12 @@
import java.util.TreeSet;
import water.TestUtil;
+import water.util.FileUtils;
import water.util.Log;
import static org.junit.Assert.assertEquals;
import static water.parser.OrcTestUtils.compareOrcAndH2OFrame;
+import static water.util.FileUtils.*;
/**
* Test suite for orc parser.
@@ -83,7 +85,7 @@ public void testParseAllOrcs() {
String fileName = allOrcFiles[fIndex];
Log.info("Orc Parser parsing " + fileName);
- File f = find_test_file_static(fileName);
+ File f = locateFile(fileName);
if (f != null && f.exists()) {
try {
diff --git a/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java b/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java
index e92747b384b..fd05c3a106f 100644
--- a/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java
+++ b/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java
@@ -3,13 +3,12 @@
import org.junit.BeforeClass;
import org.junit.Test;
import water.*;
-import water.api.ImportFilesHandler;
import water.fvec.Chunk;
import water.fvec.FileVec;
import water.fvec.Frame;
+import water.util.FileUtils;
import java.net.URI;
-import java.net.URISyntaxException;
import static junit.framework.TestCase.assertEquals;
@@ -44,7 +43,7 @@ public void testS3Import() throws Exception {
int chunkSize = (int) (v.length() / 3);
v.setChunkSize(fr, chunkSize);
long xor = new XORTask().doAll(v)._res;
- Key k2 = H2O.getPM().anyURIToKey(new URI(find_test_file("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath()));
+ Key k2 = H2O.getPM().anyURIToKey(new URI(FileUtils.getFile("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath()));
FileVec v2 = DKV.getGet(k2);
assertEquals(v2.length(), v.length());
assertVecEquals(v, v2, 0);
diff --git a/h2o-scala/src/test/scala/water/BasicTest.scala b/h2o-scala/src/test/scala/water/BasicTest.scala
index 993f533c0f4..45c0d3ba6dd 100644
--- a/h2o-scala/src/test/scala/water/BasicTest.scala
+++ b/h2o-scala/src/test/scala/water/BasicTest.scala
@@ -2,14 +2,15 @@ package water
import org.junit.{Assert, BeforeClass, Test}
import water.fvec.H2OFrame
+import water.util.FileUtils._
class BasicTest extends TestUtil {
@Test def testDataFrameLoadAPI(): Unit = {
val filename1 = "../smalldata/iris/iris_wheader.csv"
val filename2 = "../smalldata/iris/iris.csv"
- val file1 = find_test_file(filename1)
- val file2 = find_test_file(filename2)
+ val file1 = getFile(filename1)
+ val file2 = getFile(filename2)
val uri1 = file1.toURI
val uri2 = file2.toURI
// Create frames
diff --git a/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala b/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala
index 99ea2c97005..46a2727287b 100644
--- a/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala
+++ b/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala
@@ -1,6 +1,7 @@
package water.fvec
import org.junit.{Assert, Test, BeforeClass}
+import water.util.FileUtils
import water.TestUtil
import water.parser.ParseSetup
@@ -11,7 +12,7 @@ class FrameOpsTest extends TestUtil {
@Test
def testParserSetup(): Unit = {
- val irisFile = find_test_file("smalldata/iris/iris.csv")
+ val irisFile = FileUtils.getFile("smalldata/iris/iris.csv")
val f1 = new H2OFrame(irisFile)
// Default setup
val parserSetup = H2OFrame.defaultParserSetup()
@@ -34,7 +35,7 @@ class FrameOpsTest extends TestUtil {
@Test
def testApplyMethod1(): Unit = {
- val carsFile = find_test_file("smalldata/junit/cars.csv")
+ val carsFile = FileUtils.getFile("smalldata/junit/cars.csv")
val f1 = new H2OFrame(carsFile)
val subframe = f1('name)
try {
diff --git a/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala b/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala
index 9976ff543ed..f1477de38b7 100644
--- a/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala
+++ b/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala
@@ -4,13 +4,14 @@ import java.io.File
import java.util.{Date, GregorianCalendar}
import java.{lang, util}
-import org.junit.{Assert, Test, BeforeClass}
+import org.junit.{Assert, BeforeClass, Test}
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
-import water.udf.fp.PureFunctions._
-import water.udf.fp.{PureFunctions, Functions, Function}
-import water.{TestUtil, Test0}
import water.TestUtil._
import water.udf.MoreColumns._
+import water.udf.fp.PureFunctions._
+import water.udf.fp.{Function, Functions, PureFunctions}
+import water.util.FileUtils._
+import water.{Test0, TestUtil}
import scala.collection.JavaConverters._
import scala.io.Source
@@ -21,8 +22,6 @@ import scala.language.postfixOps
*/
class ScalaUdfTest extends Test0 with BeforeAndAfter with BeforeAndAfterAll {
val A_LOT: Int = 1 << 20
-
- import UdfTestBase._
override def beforeAll: Unit = stall_till_cloudsize(3)
@@ -249,7 +248,7 @@ class ScalaUdfTest extends Test0 with BeforeAndAfter with BeforeAndAfterAll {
}
test("UnfoldingColumn") {
- val file: File = UdfTestBase.getFile("smalldata/chicago/chicagoAllWeather.csv")
+ val file: File = getFile("smalldata/chicago/chicagoAllWeather.csv")
val ss = Source.fromFile(file).getLines().toList
val source: Column[lang.String] = willDrop(Strings.newColumn(ss))
diff --git a/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java b/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java
index 1e42cac035d..785f5c73587 100644
--- a/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java
+++ b/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java
@@ -1,6 +1,7 @@
package water;
import org.testng.annotations.*;
+import water.util.FileUtils;
import water.util.Log;
import java.io.*;
@@ -9,6 +10,7 @@
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.*;
+import static water.util.FileUtils.*;
public class AccuracyTestingSuite {
private String logDir;
@@ -40,8 +42,8 @@ private void accuracySuiteSetup(@org.testng.annotations.Optional("h2o-test-accur
this.logDir = logDir;
File resultsDir = null, h2oLogsDir = null;
try {
- resultsDir = new File(AccuracyTestingUtil.find_test_file_static(logDir).getCanonicalFile().toString() + "/results");
- h2oLogsDir = new File(AccuracyTestingUtil.find_test_file_static(logDir).getCanonicalFile().toString() + "/results/h2ologs");
+ resultsDir = new File(locateFile(logDir).getCanonicalFile().toString() + "/results");
+ h2oLogsDir = new File(locateFile(logDir).getCanonicalFile().toString() + "/results/h2ologs");
} catch (IOException e) {
System.out.println("Couldn't create directory.");
e.printStackTrace();
@@ -54,7 +56,7 @@ private void accuracySuiteSetup(@org.testng.annotations.Optional("h2o-test-accur
File suiteSummary;
try {
- suiteSummary = new File(AccuracyTestingUtil.find_test_file_static(logDir).getCanonicalFile().toString() +
+ suiteSummary = new File(locateFile(logDir).getCanonicalFile().toString() +
"/results/accuracySuiteSummary.log");
suiteSummary.createNewFile();
summaryLog = new PrintStream(new FileOutputStream(suiteSummary, false));
@@ -83,7 +85,7 @@ private void accuracySuiteSetup(@org.testng.annotations.Optional("h2o-test-accur
// Data sets
this.dataSetsCSVPath = dataSetsCSVPath;
- File dataSetsFile = AccuracyTestingUtil.find_test_file_static(this.dataSetsCSVPath);
+ File dataSetsFile = locateFile(this.dataSetsCSVPath);
try {
dataSetsCSVRows = Files.readAllLines(dataSetsFile.toPath(), Charset.defaultCharset());
} catch (IOException e) {
@@ -164,7 +166,7 @@ public void accuracyTest() {
try {
summaryLog.println("Reading test cases from: " + testCasesCSVPath);
- File testCasesFile = AccuracyTestingUtil.find_test_file_static(this.testCasesCSVPath);
+ File testCasesFile = getFile(testCasesCSVPath);
testCaseEntries = Files.readAllLines(testCasesFile.toPath(), Charset.defaultCharset());
}
catch (Exception e) {
diff --git a/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java b/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java
index a6f3b649d41..40ff821c496 100644
--- a/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java
+++ b/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java
@@ -1,6 +1,7 @@
package water;
import java.util.UUID;
+import static water.util.FileUtils.*;
public class AccuracyTestingUtil extends TestUtil {
private static boolean _stall_called_before = false;
@@ -18,7 +19,7 @@ public static void stall_till_cloudsize(int x, String logDir) {
if (H2O.getCloudSize() < x) {
// Leader node, where the tests execute from.
String cloudName = UUID.randomUUID().toString();
- String[] args = new String[]{"-name",cloudName,"-ice_root",find_test_file_static(logDir + "/results").
+ String[] args = new String[]{"-name",cloudName,"-ice_root", locateFile(logDir + "/results").
toString()};
H2O.main(args);
diff --git a/h2o-test-accuracy/src/test/java/water/DataSet.java b/h2o-test-accuracy/src/test/java/water/DataSet.java
index 2eebdebd26a..460e4960fa2 100644
--- a/h2o-test-accuracy/src/test/java/water/DataSet.java
+++ b/h2o-test-accuracy/src/test/java/water/DataSet.java
@@ -4,6 +4,8 @@
import java.io.IOException;
import water.fvec.Frame;
+import water.util.FileUtils;
+import static water.util.FileUtils.*;
public class DataSet {
@@ -63,6 +65,6 @@ private File makeDataSetFile(String uri) {
String[] uriTokens = uri.trim().split("/", -1);
String fileName = uriTokens[uriTokens.length - 1];
- return AccuracyTestingUtil.find_test_file_static(filePath + fileName);
+ return locateFile(filePath + fileName);
}
}