From ba4dead957a141000ead5fceda87d656af9d5480 Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Mon, 20 Feb 2017 19:08:56 -0800 Subject: [PATCH 1/2] Moved some stuff from TestUtils to FileUtils Gave the methods proper names So a bunch of tests have those minor changes Tests only --- .../DeepLearningAutoEncoderCategoricalTest.java | 3 +- .../deeplearning/DeepLearningAutoEncoderTest.java | 6 +- .../DeepLearningCheckpointReporting.java | 3 +- .../java/hex/deeplearning/DeepLearningMNIST.java | 5 +- .../hex/deeplearning/DeepLearningMissingTest.java | 3 +- .../hex/deeplearning/DeepLearningProstateTest.java | 5 +- .../DeepLearningReproducibilityTest.java | 5 +- .../hex/deeplearning/DeepLearningScoreTest.java | 3 +- .../hex/deeplearning/DeepLearningSpiralsTest.java | 3 +- .../java/hex/deeplearning/DeepLearningTest.java | 2 +- .../DeepWaterAbstractIntegrationTest.java | 17 +++-- .../deepwater/DeepWaterMXNetIntegrationTest.java | 3 +- .../test/java/hex/glm/GLMBasicTestRegression.java | 19 ++--- h2o-algos/src/test/java/hex/glrm/GLRMTest.java | 3 +- h2o-algos/src/test/java/hex/grep/GrepTest.java | 3 +- h2o-algos/src/test/java/hex/kmeans/KMeansTest.java | 2 +- .../src/test/java/hex/tree/gbm/GBMMissingTest.java | 4 +- .../test/java/hex/word2vec/WordCountTaskTest.java | 3 +- .../src/jmh/java/water/mojo/glm/GlmMojoBench.java | 4 +- .../water/mojo/glm/GlmMultinomialMojoBench.java | 5 +- h2o-core/src/main/java/water/util/FileUtils.java | 51 +++++++++++++ .../src/test/java/hex/ConfusionMatrixTest.java | 5 +- h2o-core/src/test/java/water/KVTest.java | 4 +- h2o-core/src/test/java/water/OOMTest.java | 5 +- h2o-core/src/test/java/water/TestUtil.java | 89 +++++++--------------- h2o-core/src/test/java/water/fvec/FVecTest.java | 5 +- .../test/java/water/fvec/RebalanceDatasetTest.java | 5 +- .../src/test/java/water/fvec/WordCountBigTest.java | 5 +- .../src/test/java/water/fvec/WordCountTest.java | 3 +- .../water/network/SSLSocketChannelFactoryTest.java | 7 +- .../water/parser/ParseCompressedAndXLSTest.java | 3 +- .../test/java/water/parser/ParseFolderTest.java | 3 +- .../test/java/water/parser/ParseFolderTestBig.java | 3 +- .../test/java/water/parser/ParseProgressTest.java | 3 +- .../src/test/java/water/parser/ParserTest.java | 6 +- .../src/test/java/water/parser/ParserTestARFF.java | 4 +- .../src/test/java/water/rapids/MungingTest.java | 8 +- .../src/test/java/water/rapids/RapidsTest.java | 3 +- h2o-core/src/test/java/water/udf/UdfTest.java | 1 + h2o-core/src/test/java/water/udf/UdfTestBase.java | 15 +--- .../src/test/java/water/parser/ParseTestOrc.java | 4 +- .../src/test/java/water/persist/PersistS3Test.java | 5 +- h2o-scala/src/test/scala/water/BasicTest.scala | 5 +- .../src/test/scala/water/fvec/FrameOpsTest.scala | 5 +- .../src/test/scala/water/udf/ScalaUdfTest.scala | 13 ++-- .../src/test/java/water/AccuracyTestingSuite.java | 12 +-- .../src/test/java/water/AccuracyTestingUtil.java | 3 +- h2o-test-accuracy/src/test/java/water/DataSet.java | 4 +- 48 files changed, 213 insertions(+), 167 deletions(-) diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java index 5cec285ef65..d5656ffd3c8 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java @@ -10,6 +10,7 @@ import water.fvec.NFSFileVec; import water.fvec.Vec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.Log; public class DeepLearningAutoEncoderCategoricalTest extends TestUtil { @@ -21,7 +22,7 @@ public void run() { long seed = 0xDECAF; - NFSFileVec nfs = NFSFileVec.make(find_test_file(PATH)); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(PATH)); Frame train = ParseDataset.parse(Key.make("train.hex"), nfs._key); DeepLearningParameters p = new DeepLearningParameters(); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java index 65146ea0fb0..1ec799f0b4d 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java @@ -6,8 +6,10 @@ import water.*; import water.fvec.*; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.Log; import hex.deeplearning.DeepLearningModel.DeepLearningParameters; + import java.util.HashSet; public class DeepLearningAutoEncoderTest extends TestUtil { @@ -33,9 +35,9 @@ public void run() { try { - NFSFileVec nfs = NFSFileVec.make(find_test_file(PATH)); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(PATH)); train = ParseDataset.parse(Key.make("train.hex"), nfs._key); - NFSFileVec nfs2 = NFSFileVec.make(find_test_file(PATH2)); + NFSFileVec nfs2 = NFSFileVec.make(FileUtils.locateFile(PATH2)); test = ParseDataset.parse(Key.make("test.hex"), nfs2._key); for (float sparsity_beta : new float[]{0, 0.1f}) { diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java index 67e6513cf3e..0e2b8b66af0 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java @@ -13,6 +13,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.TwoDimTable; import java.io.File; @@ -24,7 +25,7 @@ Scope.enter(); Frame frame = null; try { - File file = find_test_file("smalldata/logreg/prostate.csv"); + File file = FileUtils.locateFile("smalldata/logreg/prostate.csv"); NFSFileVec trainfv = NFSFileVec.make(file); frame = ParseDataset.parse(Key.make(), trainfv._key); DeepLearningParameters p = new DeepLearningParameters(); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMNIST.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMNIST.java index ee6cd5b1a0b..4f48a673a83 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMNIST.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMNIST.java @@ -7,6 +7,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.Log; import hex.deeplearning.DeepLearningModel.DeepLearningParameters; import java.io.File; @@ -53,8 +54,8 @@ Frame frame=null; Frame vframe=null; try { - File file = find_test_file("bigdata/laptop/mnist/train.csv.gz"); - File valid = find_test_file("bigdata/laptop/mnist/test.csv.gz"); + File file = FileUtils.locateFile("bigdata/laptop/mnist/train.csv.gz"); + File valid = FileUtils.locateFile("bigdata/laptop/mnist/test.csv.gz"); if (file != null) { NFSFileVec trainfv = NFSFileVec.make(file); frame = ParseDataset.parse(Key.make(), trainfv._key); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java index 4b3b4414c2b..21a2f1099d7 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java @@ -10,6 +10,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.FrameUtils; import water.util.Log; @@ -48,7 +49,7 @@ double loss =0; try { Scope.enter(); - NFSFileVec nfs = NFSFileVec.make(find_test_file("smalldata/junit/weather.csv")); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); data = ParseDataset.parse(Key.make("data.hex"), nfs._key); Log.info("FrameSplitting"); // Create holdout test data on clean data (before adding missing values) diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java index 7451b7d23f5..760d597ada4 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java @@ -18,6 +18,7 @@ import water.fvec.Vec; import water.parser.ParseDataset; import water.rapids.Rapids; +import water.util.FileUtils; import water.util.Log; import java.util.Arrays; @@ -48,9 +49,9 @@ public void runFraction(float fraction) { for (final int resp : responses[i]) { Frame frame=null, vframe=null; try { - NFSFileVec nfs = NFSFileVec.make(find_test_file(dataset)); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(dataset)); frame = ParseDataset.parse(Key.make(), nfs._key); - NFSFileVec vnfs = NFSFileVec.make(find_test_file(dataset)); + NFSFileVec vnfs = NFSFileVec.make(FileUtils.locateFile(dataset)); vframe = ParseDataset.parse(Key.make(), vnfs._key); boolean classification = !(i == 0 && resp == 2); String respname = frame.name(resp); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java index f00e92a4c53..1b8473a1d34 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java @@ -9,6 +9,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.FrameUtils; import water.util.Log; import hex.deeplearning.DeepLearningModel.DeepLearningParameters; @@ -19,7 +20,7 @@ @Test public void run() { - NFSFileVec ff = NFSFileVec.make(find_test_file("smalldata/junit/weather.csv")); + NFSFileVec ff = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); Frame golden = ParseDataset.parse(Key.make("golden.hex"), ff._key); DeepLearningModel mymodel = null; @@ -39,7 +40,7 @@ public void run() { double[] numbers = new double[N]; for (int repeat = 0; repeat < N; ++repeat) { try { - NFSFileVec file = NFSFileVec.make(find_test_file("smalldata/junit/weather.csv")); + NFSFileVec file = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); data = ParseDataset.parse(Key.make("data.hex"), file._key); Assert.assertTrue(TestUtil.isBitIdentical(data, golden)); //test parser consistency diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java index 86deae94dcb..e02cd415562 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java @@ -13,6 +13,7 @@ import water.fvec.Vec; import water.parser.ParseDataset; import hex.deeplearning.DeepLearningModel.DeepLearningParameters; +import water.util.FileUtils; /** * This test simulates environment @@ -27,7 +28,7 @@ @Test public void testPubDev928() { // Create rebalanced dataset Key rebalancedKey = Key.make("rebalanced"); - NFSFileVec nfs = NFSFileVec.make(find_test_file("smalldata/logreg/prostate.csv")); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/logreg/prostate.csv")); Frame fr = ParseDataset.parse(Key.make(), nfs._key); RebalanceDataSet rb = new RebalanceDataSet(fr, rebalancedKey, (int)(fr.numRows()+1)); H2O.submitTask(rb); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java index b0d639eba60..7181cfd5338 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java @@ -13,6 +13,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.Log; public class DeepLearningSpiralsTest extends TestUtil { @@ -20,7 +21,7 @@ @Test public void run() { Scope.enter(); - NFSFileVec nfs = NFSFileVec.make(find_test_file("smalldata/junit/two_spiral.csv")); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/two_spiral.csv")); Frame frame = ParseDataset.parse(Key.make(), nfs._key); Log.info(frame); int resp = frame.names().length-1; diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningTest.java index 511e93ac775..f2a3814de59 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningTest.java @@ -2135,7 +2135,7 @@ public void testMultinomialMNIST() { DeepLearningModel model = null; Scope.enter(); try { - File file = find_test_file("bigdata/laptop/mnist/train.csv.gz"); + File file = FileUtils.locateFile("bigdata/laptop/mnist/train.csv.gz"); if (file != null) { NFSFileVec trainfv = NFSFileVec.make(file); train = ParseDataset.parse(Key.make(), trainfv._key); diff --git a/h2o-algos/src/test/java/hex/deepwater/DeepWaterAbstractIntegrationTest.java b/h2o-algos/src/test/java/hex/deepwater/DeepWaterAbstractIntegrationTest.java index 4d386a0c805..f4fc5d270f9 100755 --- a/h2o-algos/src/test/java/hex/deepwater/DeepWaterAbstractIntegrationTest.java +++ b/h2o-algos/src/test/java/hex/deepwater/DeepWaterAbstractIntegrationTest.java @@ -19,6 +19,7 @@ import water.fvec.NFSFileVec; import water.fvec.Vec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.Log; import water.util.StringUtils; import water.util.TwoDimTable; @@ -748,8 +749,8 @@ public void MNISTLenet() { DeepWaterModel m = null; try { DeepWaterParameters p = new DeepWaterParameters(); - File file = find_test_file("bigdata/laptop/mnist/train.csv.gz"); - File valid = find_test_file("bigdata/laptop/mnist/test.csv.gz"); + File file = FileUtils.locateFile("bigdata/laptop/mnist/train.csv.gz"); + File valid = FileUtils.locateFile("bigdata/laptop/mnist/test.csv.gz"); if (file != null) { p._response_column = "C785"; NFSFileVec trainfv = NFSFileVec.make(file); @@ -788,8 +789,8 @@ public void MNISTSparse() { DeepWaterModel m = null; try { DeepWaterParameters p = new DeepWaterParameters(); - File file = find_test_file("bigdata/laptop/mnist/train.csv.gz"); - File valid = find_test_file("bigdata/laptop/mnist/test.csv.gz"); + File file = FileUtils.locateFile("bigdata/laptop/mnist/train.csv.gz"); + File valid = FileUtils.locateFile("bigdata/laptop/mnist/test.csv.gz"); if (file != null) { p._response_column = "C785"; NFSFileVec trainfv = NFSFileVec.make(file); @@ -826,8 +827,8 @@ public void MNISTHinton() { DeepWaterModel m = null; try { DeepWaterParameters p = new DeepWaterParameters(); - File file = find_test_file("bigdata/laptop/mnist/train.csv.gz"); - File valid = find_test_file("bigdata/laptop/mnist/test.csv.gz"); + File file = FileUtils.locateFile("bigdata/laptop/mnist/train.csv.gz"); + File valid = FileUtils.locateFile("bigdata/laptop/mnist/test.csv.gz"); if (file != null) { p._response_column = "C785"; NFSFileVec trainfv = NFSFileVec.make(file); @@ -869,7 +870,7 @@ public void Airlines() { Frame[] splits = null; try { DeepWaterParameters p = new DeepWaterParameters(); - File file = find_test_file("smalldata/airlines/allyears2k_headers.zip"); + File file = FileUtils.locateFile("smalldata/airlines/allyears2k_headers.zip"); if (file != null) { p._response_column = "IsDepDelayed"; p._ignored_columns = new String[]{"DepTime","ArrTime","Cancelled","CancellationCode","Diverted","CarrierDelay","WeatherDelay","NASDelay","SecurityDelay","LateAircraftDelay","IsArrDelayed"}; @@ -1090,7 +1091,7 @@ public void testCheckpointBackwards() { Scope.enter(); Frame frame = null; try { - File file = find_test_file("smalldata/logreg/prostate.csv"); + File file = FileUtils.locateFile("smalldata/logreg/prostate.csv"); NFSFileVec trainfv = NFSFileVec.make(file); frame = ParseDataset.parse(Key.make(), trainfv._key); DeepWaterParameters p = new DeepWaterParameters(); diff --git a/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java b/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java index 1805e613783..4934918c369 100644 --- a/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java +++ b/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java @@ -8,6 +8,7 @@ import org.junit.Ignore; import org.junit.Test; import water.parser.BufferedString; +import water.util.FileUtils; import water.util.StringUtils; import javax.imageio.ImageIO; @@ -67,7 +68,7 @@ public void inceptionPredictionMX() throws IOException { float[] mean = backend.loadMeanImage(_model, extractFile(path, "mean_224.nd")); // Turn the image into a vector of the correct size - File imgFile = find_test_file("smalldata/deepwater/imagenet/test2.jpg"); + File imgFile = FileUtils.locateFile("smalldata/deepwater/imagenet/test2.jpg"); BufferedImage img = ImageIO.read(imgFile); BufferedImage scaledImg = new BufferedImage(w, h, img.getType()); Graphics2D g2d = scaledImg.createGraphics(); diff --git a/h2o-algos/src/test/java/hex/glm/GLMBasicTestRegression.java b/h2o-algos/src/test/java/hex/glm/GLMBasicTestRegression.java index 5ac584914c6..3c701b44d5c 100644 --- a/h2o-algos/src/test/java/hex/glm/GLMBasicTestRegression.java +++ b/h2o-algos/src/test/java/hex/glm/GLMBasicTestRegression.java @@ -5,13 +5,11 @@ import hex.glm.GLMModel.GLMParameters; import hex.glm.GLMModel.GLMParameters.Family; import hex.glm.GLMModel.GLMParameters.Solver; -import junit.framework.Assert; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import water.DKV; import water.Key; -import water.MRTask; import water.TestUtil; import water.exceptions.H2OModelBuilderIllegalArgumentException; import water.fvec.Frame; @@ -19,10 +17,12 @@ import water.parser.ParseDataset; import java.io.File; +import java.io.IOException; import java.util.HashMap; -import java.util.Random; import water.fvec.*; +import static water.util.FileUtils.*; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -41,9 +41,9 @@ static Frame _airlinesMM; @BeforeClass - public static void setup() { + public static void setup() throws IOException { stall_till_cloudsize(1); - File f = find_test_file_static("smalldata/glm_test/cancar_logIn.csv"); + File f = getFile("smalldata/glm_test/cancar_logIn.csv"); assert f.exists(); NFSFileVec nfs = NFSFileVec.make(f); Key outputKey = Key.make("prostate_cat_train.hex"); @@ -52,22 +52,19 @@ public static void setup() { _canCarTrain.add("Class", (_class = _canCarTrain.remove("Class")).toCategoricalVec()); DKV.put(_canCarTrain._key, _canCarTrain); - f = find_test_file_static("smalldata/glm_test/earinf.txt"); - assert f.exists(); + f = getFile("smalldata/glm_test/earinf.txt"); nfs = NFSFileVec.make(f); outputKey = Key.make("earinf.hex"); _earinf = ParseDataset.parse(outputKey, nfs._key); DKV.put(_earinf._key, _earinf); - f = find_test_file_static("smalldata/glm_test/weighted.csv"); - assert f.exists(); + f = getFile("smalldata/glm_test/weighted.csv"); nfs = NFSFileVec.make(f); outputKey = Key.make("weighted.hex"); _weighted = ParseDataset.parse(outputKey, nfs._key); DKV.put(_weighted._key, _weighted); - f = find_test_file_static("smalldata/glm_test/upsampled.csv"); - assert f.exists(); + f = getFile("smalldata/glm_test/upsampled.csv"); nfs = NFSFileVec.make(f); outputKey = Key.make("upsampled.hex"); _upsampled = ParseDataset.parse(outputKey, nfs._key); diff --git a/h2o-algos/src/test/java/hex/glrm/GLRMTest.java b/h2o-algos/src/test/java/hex/glrm/GLRMTest.java index 7748a522aba..fc5532e56fc 100644 --- a/h2o-algos/src/test/java/hex/glrm/GLRMTest.java +++ b/h2o-algos/src/test/java/hex/glrm/GLRMTest.java @@ -20,6 +20,7 @@ import water.rapids.Rapids; import water.rapids.Val; import water.util.ArrayUtils; +import water.util.FileUtils; import water.util.FrameUtils; import water.util.Log; @@ -100,7 +101,7 @@ public static void checkLossbyCol(GLRMParameters parms, GLRMModel model) { Frame train; InputStream is; try { - is = new FileInputStream(find_test_file("bigdata/laptop/census/ACS_13_5YR_DP02_cleaned.zip")); + is = new FileInputStream(FileUtils.locateFile("bigdata/laptop/census/ACS_13_5YR_DP02_cleaned.zip")); UploadFileVec.ReadPutStats stats = new UploadFileVec.ReadPutStats(); UploadFileVec.readPut("train",is,stats); } catch (Exception e) { diff --git a/h2o-algos/src/test/java/hex/grep/GrepTest.java b/h2o-algos/src/test/java/hex/grep/GrepTest.java index 56232ceb2a8..08a33b68ba8 100644 --- a/h2o-algos/src/test/java/hex/grep/GrepTest.java +++ b/h2o-algos/src/test/java/hex/grep/GrepTest.java @@ -8,6 +8,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.fvec.Vec; +import water.util.FileUtils; import java.io.File; import static org.junit.Assert.*; @@ -22,7 +23,7 @@ //TODO: fix with original regex //String regex = "Iris-versicolor"; String regex = "ver..c\\wl[ob]r"; - File f = find_test_file("smalldata/iris/iris_wheader.csv"); + File f = FileUtils.locateFile("smalldata/iris/iris_wheader.csv"); //String regex = "(?:(\\w)\\1){5}"; //File f = new File("bigdata/text8.txt"); NFSFileVec nfs = NFSFileVec.make(f); diff --git a/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java b/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java index d128176ec2d..7e0df34228d 100755 --- a/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java +++ b/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java @@ -259,7 +259,7 @@ public int compare(final double[] a, final double[] b) { @Test @Ignore public void testCovtype() { Frame fr = null; try { - File f = find_test_file("../datasets/UCI/UCI-large/covtype/covtype.data"); + File f = FileUtils.locateFile("../datasets/UCI/UCI-large/covtype/covtype.data"); if( f==null ) return; // Ignore if large file not found NFSFileVec nfs = NFSFileVec.make(f); fr = ParseDataset.parse(Key.make(), nfs._key); diff --git a/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java b/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java index 44432a728ae..b41b51dfda6 100644 --- a/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java +++ b/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java @@ -2,7 +2,6 @@ import hex.FrameSplitter; import hex.ModelMetricsBinomial; -import hex.ScoringInfo; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -10,6 +9,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.FrameUtils; import static water.util.FrameUtils.generateNumKeys; import water.util.Log; @@ -41,7 +41,7 @@ double err=0; try { Scope.enter(); - NFSFileVec nfs = NFSFileVec.make(find_test_file("smalldata/junit/weather.csv")); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); data = ParseDataset.parse(Key.make("data.hex"), nfs._key); Log.info("FrameSplitting"); // Create holdout test data on clean data (before adding missing values) diff --git a/h2o-algos/src/test/java/hex/word2vec/WordCountTaskTest.java b/h2o-algos/src/test/java/hex/word2vec/WordCountTaskTest.java index e813c8b6c26..113577a57b4 100644 --- a/h2o-algos/src/test/java/hex/word2vec/WordCountTaskTest.java +++ b/h2o-algos/src/test/java/hex/word2vec/WordCountTaskTest.java @@ -7,6 +7,7 @@ import water.fvec.TestFrameBuilder; import water.fvec.Vec; import water.parser.BufferedString; +import static water.util.FileUtils.*; import water.util.IcedLong; import java.util.Map; @@ -55,7 +56,7 @@ else if (b < 5) @Test public void testWordCountText8() { String fName = "bigdata/laptop/text8.gz"; - assumeThat("text8 data available", find_test_file_static(fName), is(notNullValue())); // only run if text8 is present + assumeThat("text8 data available", locateFile(fName), is(notNullValue())); // only run if text8 is present Frame fr = parse_test_file(fName, "NA", 0, new byte[]{Vec.T_STR}); try { Map counts = new WordCountTask().doAll(fr.vec(0))._counts; diff --git a/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java b/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java index 10a5c74f6bb..bdad77af5d8 100644 --- a/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java +++ b/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java @@ -7,11 +7,11 @@ import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import static water.util.FileUtils.*; import java.io.*; import java.util.concurrent.TimeUnit; -import static water.TestUtil.find_test_file_static; import static water.mojo.glm.GlmMojoBenchHelper.*; /** @@ -41,7 +41,7 @@ @Setup public void setup() throws IOException { - File f = find_test_file_static("smalldata/airlines/allyears2k.zip"); + File f = locateFile("smalldata/airlines/allyears2k.zip"); mojo = (GlmMojoModel) loadMojo("airlines"); diff --git a/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java b/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java index f6a43f5dcf5..b310e5cf51e 100644 --- a/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java +++ b/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java @@ -7,12 +7,13 @@ import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import water.util.FileUtils; import java.io.*; import java.util.concurrent.TimeUnit; -import static water.TestUtil.find_test_file_static; import static water.mojo.glm.GlmMojoBenchHelper.*; +import static water.util.FileUtils.*; /** * GLM MOJO scoring benchmark (multinomial) @@ -41,7 +42,7 @@ @Setup public void setup() throws IOException { - File f = find_test_file_static("smalldata/flow_examples/mnist/test.csv.gz"); + File f = locateFile("smalldata/flow_examples/mnist/test.csv.gz"); mojo = (GlmMultinomialMojoModel) loadMojo("mnist"); diff --git a/h2o-core/src/main/java/water/util/FileUtils.java b/h2o-core/src/main/java/water/util/FileUtils.java index 91886bdb7c4..9cf4387eb62 100644 --- a/h2o-core/src/main/java/water/util/FileUtils.java +++ b/h2o-core/src/main/java/water/util/FileUtils.java @@ -64,6 +64,57 @@ else if (file.isDirectory()) { return false; } + + /** Hunt for files in likely places. Null if cannot find. + * @param fname filename + * @return Found file or null */ + public static File locateFile(String fname) { + // When run from eclipse, the working directory is different. + // Try pointing at another likely place + File file = new File(fname); + if( !file.exists() ) + file = new File("target/" + fname); + if( !file.exists() ) + file = new File("../" + fname); + if( !file.exists() ) + file = new File("../../" + fname); + if( !file.exists() ) + file = new File("../target/" + fname); + if( !file.exists() ) + file = new File(StringUtils.expandPath(fname)); + if( !file.exists() ) + file = null; + return file; + } + + private static void check(boolean cond, String msg) throws IOException{ + if (!cond) throw new IOException(msg); + } + + private static void checkFileEntry(String name, File file) throws IOException { + check(file != null, "File not found: " + name); + check(file.exists(), "File should exist: " + name); + } + + public static void checkFile(File file, String name) throws IOException { + checkFileEntry(name, file); + check(file.isFile(), "Expected a file: " + name); + check(file.canRead(), "Expected a readable file: " + name); + } + + public static File getFile(String fname) throws IOException { + File f = locateFile(fname); + checkFile(f, fname); + return f; + } + + public static File[] contentsOf(File folder, String name) throws IOException { + checkFileEntry(name, folder); + if (!folder.isDirectory()) throw new IOException("Expected a folder: " + name); + File[] files = folder.listFiles(); + if (files == null) throw new IOException("Cannot read folder: " + folder); + return files; + } /** Transform given key to a string which can be used as a file name. */ public static String keyToFileName(Key k) { diff --git a/h2o-core/src/test/java/hex/ConfusionMatrixTest.java b/h2o-core/src/test/java/hex/ConfusionMatrixTest.java index 2c7eb3803af..a3162c2e256 100644 --- a/h2o-core/src/test/java/hex/ConfusionMatrixTest.java +++ b/h2o-core/src/test/java/hex/ConfusionMatrixTest.java @@ -8,6 +8,7 @@ import water.TestUtil; import water.fvec.Frame; import water.util.ArrayUtils; +import water.util.FileUtils; import water.util.VecUtils; import static water.util.FrameUtils.parseFrame; @@ -196,8 +197,8 @@ public void testDifferentDomainsNumericVectors() { private void simpleCMTest(String f1, String f2, String[] expectedActualDomain, String[] expectedPredictDomain, String[] expectedDomain, double[][] expectedCM, boolean debug) { try { - Frame v1 = parseFrame(Key.make("v1.hex"), find_test_file(f1)); - Frame v2 = parseFrame(Key.make("v2.hex"), find_test_file(f2)); + Frame v1 = parseFrame(Key.make("v1.hex"), FileUtils.locateFile(f1)); + Frame v2 = parseFrame(Key.make("v2.hex"), FileUtils.locateFile(f2)); if (!v1.isCompatible(v2)) { Frame old = null; v2 = new Frame(v1.makeCompatible(old = v2)); diff --git a/h2o-core/src/test/java/water/KVTest.java b/h2o-core/src/test/java/water/KVTest.java index 8f57d2e5710..4624faf9c22 100644 --- a/h2o-core/src/test/java/water/KVTest.java +++ b/h2o-core/src/test/java/water/KVTest.java @@ -5,8 +5,8 @@ import java.io.File; import water.fvec.Chunk; -import water.fvec.Frame; import water.fvec.NFSFileVec; +import water.util.FileUtils; import water.util.UnsafeUtils; public class KVTest extends TestUtil { @@ -126,7 +126,7 @@ long start = System.currentTimeMillis(); NFSFileVec nfs = null; try { - File file = find_test_file("build/h2o-core.jar"); + File file = FileUtils.locateFile("build/h2o-core.jar"); if( file == null ) return; // Nothing to test // Return a Key mapping to a NFSFileVec over the file nfs = NFSFileVec.make(file); diff --git a/h2o-core/src/test/java/water/OOMTest.java b/h2o-core/src/test/java/water/OOMTest.java index 59436d2b8e5..86331df99fd 100644 --- a/h2o-core/src/test/java/water/OOMTest.java +++ b/h2o-core/src/test/java/water/OOMTest.java @@ -7,6 +7,7 @@ import org.junit.*; import water.fvec.Frame; import water.fvec.Vec; +import static water.util.FileUtils.*; import water.util.Log; @Ignore @@ -93,8 +94,8 @@ @Test @Ignore public void testParseMemoryStress() { // "bigdata directory is not always available" - if( find_test_file_static("bigdata/laptop/usecases/cup98LRN_z.csv") == null ) return; - if( find_test_file_static("bigdata/laptop/usecases/cup98VAL_z.csv") == null ) return; + if( locateFile("bigdata/laptop/usecases/cup98LRN_z.csv") == null ) return; + if( locateFile("bigdata/laptop/usecases/cup98VAL_z.csv") == null ) return; ArrayList frames = new ArrayList<>(); File ice = new File(water.H2O.ICE_ROOT.toString(),"ice" + water.H2O.API_PORT); String[] dirs = ice.list(); diff --git a/h2o-core/src/test/java/water/TestUtil.java b/h2o-core/src/test/java/water/TestUtil.java index 838601644e7..6d7ddc3ed04 100644 --- a/h2o-core/src/test/java/water/TestUtil.java +++ b/h2o-core/src/test/java/water/TestUtil.java @@ -1,6 +1,5 @@ package water; -import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Ignore; @@ -13,10 +12,8 @@ import water.parser.DefaultParserProviders; import water.parser.ParseDataset; import water.parser.ParseSetup; -import water.util.Log; -import water.util.StringUtils; +import water.util.*; import water.util.Timer; -import water.util.TwoDimTable; import java.io.File; import java.io.IOException; @@ -24,9 +21,7 @@ import java.lang.reflect.Method; import java.util.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; @Ignore("Support for tests, but no actual tests here") public class TestUtil extends Iced { @@ -192,28 +187,6 @@ public void evaluate() throws Throwable {} // ==== Data Frame Creation Utilities ==== - /** Hunt for test files in likely places. Null if cannot find. - * @param fname Test filename - * @return Found file or null */ - public static File find_test_file_static(String fname) { - // When run from eclipse, the working directory is different. - // Try pointing at another likely place - File file = new File(fname); - if( !file.exists() ) - file = new File("target/" + fname); - if( !file.exists() ) - file = new File("../" + fname); - if( !file.exists() ) - file = new File("../../" + fname); - if( !file.exists() ) - file = new File("../target/" + fname); - if( !file.exists() ) - file = new File(StringUtils.expandPath(fname)); - if( !file.exists() ) - file = null; - return file; - } - /** Compare 2 frames * @param fr1 Frame * @param fr2 Frame @@ -238,29 +211,21 @@ public static boolean isBitIdentical(Frame fr1, Frame fr2) { return isIdenticalUpToRelTolerance(fr1,fr2,0); } - /** Hunt for test files in likely places. Null if cannot find. - * @param fname Test filename - * @return Found file or null */ - protected File find_test_file(String fname) { - return find_test_file_static(fname); - } - - private static void checkFileEntry(String name, File file) { - assertNotNull("File not found: " + name, file); - assertTrue("File should exist: " + name, file.exists()); - } - public static void checkFile(String name, File file) { - checkFileEntry(name, file); - assertTrue("Expected a readable file: " + name, file.canRead()); + try { + FileUtils.checkFile(file, name); + } catch (IOException ioe) { + fail(ioe.getMessage()); + } } - static File[] checkFolder(String name, File folder) { - checkFileEntry(name, folder); - assertTrue("Expected a folder: " + name, folder.isDirectory()); - File[] files = folder.listFiles(); - assertNotNull("No files found in " + folder, files); - return files; + static File[] contentsOf(String name, File folder) { + try { + return FileUtils.contentsOf(folder, name); + } catch (IOException ioe) { + fail(ioe.getMessage()); + return null; + } } /** Find & parse a CSV file. NPE if file not found. @@ -268,23 +233,25 @@ public static void checkFile(String name, File file) { * @return Frame or NPE */ public static Frame parse_test_file( String fname ) { return parse_test_file(Key.make(),fname); } public static Frame parse_test_file( Key outputKey, String fname) { - File f = find_test_file_static(fname); - checkFile(fname, f); - NFSFileVec nfs = NFSFileVec.make(f); + NFSFileVec nfs = makeNfsFileVec(fname); return ParseDataset.parse(outputKey, nfs._key); } + static NFSFileVec makeNfsFileVec(String fname) { + File f = FileUtils.locateFile(fname); + checkFile(fname, f); + return NFSFileVec.make(f); + } + protected Frame parse_test_file( Key outputKey, String fname , boolean guessSetup) { - File f = find_test_file(fname); + File f = FileUtils.locateFile(fname); checkFile(fname, f); NFSFileVec nfs = NFSFileVec.make(f); return ParseDataset.parse(outputKey, new Key[]{nfs._key}, true, ParseSetup.guessSetup(new Key[]{nfs._key},false,1)); } protected Frame parse_test_file( String fname, String na_string, int check_header, byte[] column_types ) { - File f = find_test_file_static(fname); - checkFile(fname, f); - NFSFileVec nfs = NFSFileVec.make(f); + NFSFileVec nfs = makeNfsFileVec(fname); Key[] res = {nfs._key}; @@ -317,8 +284,8 @@ protected Frame parse_test_file( String fname, String na_string, int check_heade * @param fname Test filename * @return Frame or NPE */ protected Frame parse_test_folder( String fname ) { - File folder = find_test_file(fname); - File[] files = checkFolder(fname, folder); + File folder = FileUtils.locateFile(fname); + File[] files = contentsOf(fname, folder); Arrays.sort(files); ArrayList keys = new ArrayList<>(); for( File f : files ) @@ -338,8 +305,8 @@ protected Frame parse_test_folder( String fname ) { * @return */ protected static Frame parse_test_folder( String fname, String na_string, int check_header, byte[] column_types ) { - File folder = find_test_file_static(fname); - File[] files = checkFolder(fname, folder); + File folder = FileUtils.locateFile(fname); + File[] files = contentsOf(fname, folder); Arrays.sort(files); ArrayList keys = new ArrayList<>(); for( File f : files ) @@ -756,7 +723,7 @@ public Frame prepare() { public void done(Frame frame) { if (generatedFile != null) { generatedFile.deleteOnExit(); - FileUtils.deleteQuietly(generatedFile); + org.apache.commons.io.FileUtils.deleteQuietly(generatedFile); } } } diff --git a/h2o-core/src/test/java/water/fvec/FVecTest.java b/h2o-core/src/test/java/water/fvec/FVecTest.java index ab4dfff4ab8..64a97ccffc9 100644 --- a/h2o-core/src/test/java/water/fvec/FVecTest.java +++ b/h2o-core/src/test/java/water/fvec/FVecTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import water.*; import water.util.ArrayUtils; +import water.util.FileUtils; import java.io.File; @@ -47,7 +48,7 @@ public static Key makeByteVec(Key k, String... data) { // ========================================================================== @Test public void testBasicCRUD() { // Make and insert a FileVec to the global store - File file = find_test_file("./smalldata/junit/cars.csv"); + File file = FileUtils.locateFile("./smalldata/junit/cars.csv"); NFSFileVec nfs = NFSFileVec.make(file); int sum = ArrayUtils.sum(new ByteHisto().doAll(nfs)._x); assertEquals(file.length(),sum); @@ -107,7 +108,7 @@ public static Key makeByteVec(Key k, String... data) { // Test making a appendable vector from a plain vector @Test public void testNewVec() { // Make and insert a File8Vec to the global store - File file = find_test_file("./smalldata/junit/cars.csv"); + File file = FileUtils.locateFile("./smalldata/junit/cars.csv"); NFSFileVec nfs = NFSFileVec.make(file); Vec res = new TestNewVec().doAll(new byte[]{Vec.T_NUM},nfs).outputFrame(new String[]{"v"},new String[][]{null}).anyVec(); assertEquals(nfs.at8(0)+1,res.at8(0)); diff --git a/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java b/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java index ee01aec7cbf..f247849bdd2 100644 --- a/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java +++ b/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java @@ -5,6 +5,7 @@ import water.*; import water.parser.ParseDataset; +import water.util.FileUtils; import water.util.FrameUtils; import water.util.Log; @@ -12,8 +13,8 @@ @BeforeClass public static void setup() { stall_till_cloudsize(1); } @Test public void testProstate(){ NFSFileVec[] nfs = new NFSFileVec[]{ - NFSFileVec.make(find_test_file("smalldata/logreg/prostate.csv")), - NFSFileVec.make(find_test_file("smalldata/covtype/covtype.20k.data"))}; + NFSFileVec.make(FileUtils.locateFile("smalldata/logreg/prostate.csv")), + NFSFileVec.make(FileUtils.locateFile("smalldata/covtype/covtype.20k.data"))}; //NFSFileVec.make(find_test_file("bigdata/laptop/usecases/cup98VAL_z.csv"))}; for (NFSFileVec fv : nfs) { Frame fr = ParseDataset.parse(Key.make(), fv._key); diff --git a/h2o-core/src/test/java/water/fvec/WordCountBigTest.java b/h2o-core/src/test/java/water/fvec/WordCountBigTest.java index 27fc231bf0a..ff2b173e86c 100644 --- a/h2o-core/src/test/java/water/fvec/WordCountBigTest.java +++ b/h2o-core/src/test/java/water/fvec/WordCountBigTest.java @@ -1,14 +1,15 @@ package water.fvec; import org.junit.*; +import water.util.FileUtils; import java.io.*; public class WordCountBigTest extends WordCountTest { @Test public void testWordCountWiki() throws IOException { String best = "/home/0xdiag/datasets/wiki.xml"; - File file = find_test_file(best); - if( file==null ) file = find_test_file("../datasets/Wiki_20130805.xml"); + File file = FileUtils.locateFile(best); + if( file==null ) file = FileUtils.locateFile("../datasets/Wiki_20130805.xml"); if( file==null ) throw new FileNotFoundException(best); doWordCount(file); } diff --git a/h2o-core/src/test/java/water/fvec/WordCountTest.java b/h2o-core/src/test/java/water/fvec/WordCountTest.java index d33da7eb9ea..bd177e3d29e 100644 --- a/h2o-core/src/test/java/water/fvec/WordCountTest.java +++ b/h2o-core/src/test/java/water/fvec/WordCountTest.java @@ -8,13 +8,14 @@ import java.util.concurrent.atomic.AtomicLong; import water.*; import water.nbhm.NonBlockingHashMap; +import water.util.FileUtils; public class WordCountTest extends TestUtil { @BeforeClass static public void setup() { stall_till_cloudsize(1); } // ========================================================================== @Test public void testWordCount() throws IOException { - File file = find_test_file("./smalldata/junit/cars.csv"); + File file = FileUtils.locateFile("./smalldata/junit/cars.csv"); doWordCount(file); } diff --git a/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java b/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java index 8a66e195467..063f5f1bd80 100644 --- a/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java +++ b/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java @@ -1,7 +1,7 @@ package water.network; import org.junit.Test; -import water.TestUtil; +import water.util.FileUtils; import javax.net.ssl.SSLException; import java.io.IOException; @@ -14,6 +14,7 @@ import java.util.concurrent.CyclicBarrier; import static org.junit.Assert.*; +import static water.util.FileUtils.*; public class SSLSocketChannelFactoryTest { @@ -23,9 +24,9 @@ public void shouldHandshake() throws IOException, SSLContextException, BrokenBarrierException, InterruptedException { SSLProperties props = new SSLProperties(); props.put("h2o_ssl_protocol", SecurityUtils.defaultTLSVersion()); - props.put("h2o_ssl_jks_internal", TestUtil.find_test_file_static("src/test/resources/keystore.jks").getPath()); + props.put("h2o_ssl_jks_internal", locateFile("src/test/resources/keystore.jks").getPath()); props.put("h2o_ssl_jks_password", "password"); - props.put("h2o_ssl_jts", TestUtil.find_test_file_static("src/test/resources/cacerts.jks").getPath()); + props.put("h2o_ssl_jts", locateFile("src/test/resources/cacerts.jks").getPath()); props.put("h2o_ssl_jts_password", "password"); final SSLSocketChannelFactory factory = new SSLSocketChannelFactory(props); diff --git a/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java b/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java index 9cbaf2a1725..1ca58cdd5d1 100644 --- a/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java +++ b/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java @@ -10,6 +10,7 @@ import water.fvec.Vec; import water.fvec.Frame; import water.fvec.NFSFileVec; +import water.util.FileUtils; public class ParseCompressedAndXLSTest extends TestUtil { @BeforeClass static public void setup() { stall_till_cloudsize(5); } @@ -50,7 +51,7 @@ @Test public void testXLSBadArgs(){ Frame k1 = null; try { - File f = find_test_file("smalldata/airlines/AirlinesTest.csv.zip"); + File f = FileUtils.locateFile("smalldata/airlines/AirlinesTest.csv.zip"); NFSFileVec nfs = NFSFileVec.make(f); byte[] ctypes = new byte[12]; for(int i=0; i < 12; i++) ctypes[i] = Vec.T_NUM; diff --git a/h2o-core/src/test/java/water/parser/ParseFolderTest.java b/h2o-core/src/test/java/water/parser/ParseFolderTest.java index 8d25b2ba621..e903693992b 100644 --- a/h2o-core/src/test/java/water/parser/ParseFolderTest.java +++ b/h2o-core/src/test/java/water/parser/ParseFolderTest.java @@ -7,6 +7,7 @@ import water.TestUtil; import water.fvec.Frame; import water.fvec.NFSFileVec; +import water.util.FileUtils; public class ParseFolderTest extends TestUtil { @BeforeClass static public void setup() { stall_till_cloudsize(5); } @@ -24,7 +25,7 @@ } @Test public void testSameFile() { - File f = find_test_file("smalldata/iris/iris_wheader.csv"); + File f = FileUtils.locateFile("smalldata/iris/iris_wheader.csv"); NFSFileVec nfs1 = NFSFileVec.make(f); NFSFileVec nfs2 = NFSFileVec.make(f); Frame fr = null; diff --git a/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java b/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java index 4ca24a06e5f..7a6cf577ba6 100644 --- a/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java +++ b/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java @@ -9,6 +9,7 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.fvec.Vec; +import water.util.FileUtils; public class ParseFolderTestBig extends TestUtil { @BeforeClass static public void setup() { stall_till_cloudsize(1); } @@ -53,7 +54,7 @@ public void testBIGSVM() { String fname = "bigdata/cust_K/1m.svm"; Frame k1 = null; try { - File f = find_test_file(fname); + File f = FileUtils.locateFile(fname); assert f != null && f.exists():" file not found: " + fname; NFSFileVec nfs = NFSFileVec.make(f); Job job = ParseDataset.parse(Key.make("BIGSVM.hex"),new Key[]{nfs._key},true,ParseSetup.guessSetup(new Key[]{nfs._key}, false, ParseSetup.GUESS_HEADER),false)._job; diff --git a/h2o-core/src/test/java/water/parser/ParseProgressTest.java b/h2o-core/src/test/java/water/parser/ParseProgressTest.java index a0b49c47dde..61a3d4aefa6 100644 --- a/h2o-core/src/test/java/water/parser/ParseProgressTest.java +++ b/h2o-core/src/test/java/water/parser/ParseProgressTest.java @@ -7,6 +7,7 @@ import water.TestUtil; import water.fvec.Frame; import water.util.FileIntegrityChecker; +import water.util.FileUtils; public class ParseProgressTest extends TestUtil { // Attempt a multi-jvm parse of covtype. @@ -15,7 +16,7 @@ String[] covtype_locations = new String[]{"../datasets/UCI/UCI-large/covtype/covtype.data", "../../datasets/UCI/UCI-large/covtype/covtype.data", "../datasets/UCI/UCI-large/covtype/covtype.data.gz", "../demo/UCI-large/covtype/covtype.data", }; File f = null; for( String covtype_location : covtype_locations ) { - f = find_test_file( covtype_location ); + f = FileUtils.locateFile(covtype_location); if( f != null && f.exists() ) break; } diff --git a/h2o-core/src/test/java/water/parser/ParserTest.java b/h2o-core/src/test/java/water/parser/ParserTest.java index e8122ae0e7c..13d3a39b4f2 100644 --- a/h2o-core/src/test/java/water/parser/ParserTest.java +++ b/h2o-core/src/test/java/water/parser/ParserTest.java @@ -9,9 +9,9 @@ import water.*; import water.api.schemas3.ParseSetupV3; import water.fvec.*; +import water.util.FileUtils; import water.util.Log; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -845,7 +845,7 @@ static void testParsed(Frame fr, double[][] expected, int len) { }) { try { Log.info("Trying to parse " + f); - NFSFileVec nfs = NFSFileVec.make(find_test_file(f)); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(f)); Frame fr = ParseDataset.parse(Key.make(), new Key[]{nfs._key}, delete_on_done, true /*single quote*/, check_header); fr.delete(); } catch (Throwable t) { @@ -886,7 +886,7 @@ public void testPubDev2897() { @Test public void testParserRespectsSpecifiedColNum() { - Vec fv = NFSFileVec.make(find_test_file("smalldata/jira/runit_pubdev_3590_unexpected_column.csv")); + Vec fv = NFSFileVec.make(FileUtils.locateFile("smalldata/jira/runit_pubdev_3590_unexpected_column.csv")); Key fkey = Key.make("data4cols"); try { Key[] keys = new Key[]{fv._key}; diff --git a/h2o-core/src/test/java/water/parser/ParserTestARFF.java b/h2o-core/src/test/java/water/parser/ParserTestARFF.java index 7b011baba04..bd78fec5997 100644 --- a/h2o-core/src/test/java/water/parser/ParserTestARFF.java +++ b/h2o-core/src/test/java/water/parser/ParserTestARFF.java @@ -11,6 +11,8 @@ import water.fvec.Frame; import water.fvec.NFSFileVec; import water.fvec.Vec; +import water.util.FileUtils; + import static water.parser.ParserTest.makeByteVec; import java.util.Arrays; @@ -91,7 +93,7 @@ private void testColNames(String[] dataset, String[] exp, int len, String sep) { * @param fname Test filename * @return Frame or NPE */ private Frame parse_test_file_single_quotes( String fname ) { - NFSFileVec nfs = NFSFileVec.make(find_test_file(fname)); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(fname)); return ParseDataset.parse(Key.make(), new Key[]{nfs._key}, true, true /*single quote*/, ParseSetup.GUESS_HEADER); } diff --git a/h2o-core/src/test/java/water/rapids/MungingTest.java b/h2o-core/src/test/java/water/rapids/MungingTest.java index e48cb8dc901..fd43a8b0514 100644 --- a/h2o-core/src/test/java/water/rapids/MungingTest.java +++ b/h2o-core/src/test/java/water/rapids/MungingTest.java @@ -6,7 +6,7 @@ import water.*; import water.fvec.*; import water.parser.ParseDataset; -import water.util.Log; +import water.util.FileUtils; import java.io.InputStream; import java.io.OutputStream; @@ -32,7 +32,7 @@ private void copyStream(OutputStream os, InputStream is, final int buffer_size) } @Ignore @Test public void run2() throws Exception { System.out.println("Running run2 ..."); - NFSFileVec nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/step1.csv")); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("/home/mdowle/devtestdata/step1.csv")); Frame frame = ParseDataset.parse(Key.make(), nfs._key); // look into parse() to manip column types System.out.println("Loaded file, now calling Query ..."); // new RadixOrder(frame, true, new int[] {0,1}); // group by 0=id, 1=date and sum 3 == quantity @@ -43,12 +43,12 @@ private void copyStream(OutputStream os, InputStream is, final int buffer_size) @Ignore @Test public void run3() throws Exception { System.out.println("Running run3 ..."); - NFSFileVec nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/step1_subset.csv")); + NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("/home/mdowle/devtestdata/step1_subset.csv")); //NFSFileVec nfs = NFSFileVec.make(find_test_file("/users/arno/devtestdata/step1_subset.csv")); Frame leftFrame = ParseDataset.parse(Key.make(), nfs._key); //nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/fullsize.csv")); - nfs = NFSFileVec.make(find_test_file("/home/mdowle/devtestdata/fullsize.csv")); + nfs = NFSFileVec.make(FileUtils.locateFile("/home/mdowle/devtestdata/fullsize.csv")); //nfs = NFSFileVec.make(find_test_file("/users/arno/devtestdata/fullsize.csv")); Frame rightFrame = ParseDataset.parse(Key.make(), nfs._key); // look into parse() to manip column types diff --git a/h2o-core/src/test/java/water/rapids/RapidsTest.java b/h2o-core/src/test/java/water/rapids/RapidsTest.java index 5185a6c52eb..c167cdfcee4 100644 --- a/h2o-core/src/test/java/water/rapids/RapidsTest.java +++ b/h2o-core/src/test/java/water/rapids/RapidsTest.java @@ -14,6 +14,7 @@ import water.rapids.ast.params.AstStr; import water.rapids.vals.ValFrame; import water.util.ArrayUtils; +import water.util.FileUtils; import water.util.Log; import java.io.File; @@ -461,7 +462,7 @@ static boolean checkSaneFrame_impl() { parse_test_file(Key.make("weather.hex"),"smalldata/chicago/chicagoAllWeather.csv"); parse_test_file(Key.make( "crimes.hex"),"smalldata/chicago/chicagoCrimes10k.csv.zip"); String fname = "smalldata/chicago/chicagoCensus.csv"; - File f = find_test_file(fname); + File f = FileUtils.locateFile(fname); assert f != null && f.exists():" file not found: " + fname; NFSFileVec nfs = NFSFileVec.make(f); ParseSetup ps = ParseSetup.guessSetup(new Key[]{nfs._key}, false, 1); diff --git a/h2o-core/src/test/java/water/udf/UdfTest.java b/h2o-core/src/test/java/water/udf/UdfTest.java index b1fed577767..0085f1389b9 100644 --- a/h2o-core/src/test/java/water/udf/UdfTest.java +++ b/h2o-core/src/test/java/water/udf/UdfTest.java @@ -17,6 +17,7 @@ import static water.udf.specialized.Dates.*; import static water.udf.specialized.Doubles.*; import static water.udf.specialized.Strings.*; +import static water.util.FileUtils.*; /** * Test for UDF */ diff --git a/h2o-core/src/test/java/water/udf/UdfTestBase.java b/h2o-core/src/test/java/water/udf/UdfTestBase.java index df8eea3da8a..20bdd60a38c 100644 --- a/h2o-core/src/test/java/water/udf/UdfTestBase.java +++ b/h2o-core/src/test/java/water/udf/UdfTestBase.java @@ -4,16 +4,15 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import water.H2OStarter; -import water.Iced; import water.Scope; import water.TestUtil; import water.fvec.NFSFileVec; import water.fvec.Vec; +import water.util.FileUtils; import java.io.File; +import java.io.IOException; import java.lang.reflect.Method; -import static water.TestUtil.*; /** * All test functionality specific for udf (not actually), @@ -52,17 +51,11 @@ public void bye() { } - public static Vec loadFile(String fname) { - File f = getFile(fname); + public static Vec loadFile(String fname) throws IOException { + File f = FileUtils.getFile(fname); return NFSFileVec.make(f); } - public static File getFile(String fname) { - File f = find_test_file_static(fname); - checkFile(fname, f); - return f; - } - // the following code exists or else gradlew will complain; also, it checks assertions @Test public void testAssertionsEnabled() throws Exception { diff --git a/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java b/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java index 50fbbe0d826..610fe5cc424 100644 --- a/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java +++ b/h2o-parsers/h2o-orc-parser/src/test/java/water/parser/ParseTestOrc.java @@ -10,10 +10,12 @@ import java.util.TreeSet; import water.TestUtil; +import water.util.FileUtils; import water.util.Log; import static org.junit.Assert.assertEquals; import static water.parser.OrcTestUtils.compareOrcAndH2OFrame; +import static water.util.FileUtils.*; /** * Test suite for orc parser. @@ -83,7 +85,7 @@ public void testParseAllOrcs() { String fileName = allOrcFiles[fIndex]; Log.info("Orc Parser parsing " + fileName); - File f = find_test_file_static(fileName); + File f = locateFile(fileName); if (f != null && f.exists()) { try { diff --git a/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java b/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java index e92747b384b..2c3b54f2538 100644 --- a/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java +++ b/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java @@ -3,13 +3,12 @@ import org.junit.BeforeClass; import org.junit.Test; import water.*; -import water.api.ImportFilesHandler; import water.fvec.Chunk; import water.fvec.FileVec; import water.fvec.Frame; +import water.util.FileUtils; import java.net.URI; -import java.net.URISyntaxException; import static junit.framework.TestCase.assertEquals; @@ -44,7 +43,7 @@ public void testS3Import() throws Exception { int chunkSize = (int) (v.length() / 3); v.setChunkSize(fr, chunkSize); long xor = new XORTask().doAll(v)._res; - Key k2 = H2O.getPM().anyURIToKey(new URI(find_test_file("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath())); + Key k2 = H2O.getPM().anyURIToKey(new URI(FileUtils.locateFile("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath())); FileVec v2 = DKV.getGet(k2); assertEquals(v2.length(), v.length()); assertVecEquals(v, v2, 0); diff --git a/h2o-scala/src/test/scala/water/BasicTest.scala b/h2o-scala/src/test/scala/water/BasicTest.scala index 993f533c0f4..b6244596e37 100644 --- a/h2o-scala/src/test/scala/water/BasicTest.scala +++ b/h2o-scala/src/test/scala/water/BasicTest.scala @@ -2,14 +2,15 @@ package water import org.junit.{Assert, BeforeClass, Test} import water.fvec.H2OFrame +import water.util.FileUtils._ class BasicTest extends TestUtil { @Test def testDataFrameLoadAPI(): Unit = { val filename1 = "../smalldata/iris/iris_wheader.csv" val filename2 = "../smalldata/iris/iris.csv" - val file1 = find_test_file(filename1) - val file2 = find_test_file(filename2) + val file1 = locateFile(filename1) + val file2 = locateFile(filename2) val uri1 = file1.toURI val uri2 = file2.toURI // Create frames diff --git a/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala b/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala index 99ea2c97005..1a4ea2a037c 100644 --- a/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala +++ b/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala @@ -1,6 +1,7 @@ package water.fvec import org.junit.{Assert, Test, BeforeClass} +import water.util.FileUtils import water.TestUtil import water.parser.ParseSetup @@ -11,7 +12,7 @@ class FrameOpsTest extends TestUtil { @Test def testParserSetup(): Unit = { - val irisFile = find_test_file("smalldata/iris/iris.csv") + val irisFile = FileUtils.locateFile("smalldata/iris/iris.csv") val f1 = new H2OFrame(irisFile) // Default setup val parserSetup = H2OFrame.defaultParserSetup() @@ -34,7 +35,7 @@ class FrameOpsTest extends TestUtil { @Test def testApplyMethod1(): Unit = { - val carsFile = find_test_file("smalldata/junit/cars.csv") + val carsFile = FileUtils.locateFile("smalldata/junit/cars.csv") val f1 = new H2OFrame(carsFile) val subframe = f1('name) try { diff --git a/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala b/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala index 9976ff543ed..f1477de38b7 100644 --- a/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala +++ b/h2o-scala/src/test/scala/water/udf/ScalaUdfTest.scala @@ -4,13 +4,14 @@ import java.io.File import java.util.{Date, GregorianCalendar} import java.{lang, util} -import org.junit.{Assert, Test, BeforeClass} +import org.junit.{Assert, BeforeClass, Test} import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} -import water.udf.fp.PureFunctions._ -import water.udf.fp.{PureFunctions, Functions, Function} -import water.{TestUtil, Test0} import water.TestUtil._ import water.udf.MoreColumns._ +import water.udf.fp.PureFunctions._ +import water.udf.fp.{Function, Functions, PureFunctions} +import water.util.FileUtils._ +import water.{Test0, TestUtil} import scala.collection.JavaConverters._ import scala.io.Source @@ -21,8 +22,6 @@ import scala.language.postfixOps */ class ScalaUdfTest extends Test0 with BeforeAndAfter with BeforeAndAfterAll { val A_LOT: Int = 1 << 20 - - import UdfTestBase._ override def beforeAll: Unit = stall_till_cloudsize(3) @@ -249,7 +248,7 @@ class ScalaUdfTest extends Test0 with BeforeAndAfter with BeforeAndAfterAll { } test("UnfoldingColumn") { - val file: File = UdfTestBase.getFile("smalldata/chicago/chicagoAllWeather.csv") + val file: File = getFile("smalldata/chicago/chicagoAllWeather.csv") val ss = Source.fromFile(file).getLines().toList val source: Column[lang.String] = willDrop(Strings.newColumn(ss)) diff --git a/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java b/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java index 1e42cac035d..c05f82b4618 100644 --- a/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java +++ b/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java @@ -1,6 +1,7 @@ package water; import org.testng.annotations.*; +import water.util.FileUtils; import water.util.Log; import java.io.*; @@ -9,6 +10,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.*; +import static water.util.FileUtils.*; public class AccuracyTestingSuite { private String logDir; @@ -40,8 +42,8 @@ private void accuracySuiteSetup(@org.testng.annotations.Optional("h2o-test-accur this.logDir = logDir; File resultsDir = null, h2oLogsDir = null; try { - resultsDir = new File(AccuracyTestingUtil.find_test_file_static(logDir).getCanonicalFile().toString() + "/results"); - h2oLogsDir = new File(AccuracyTestingUtil.find_test_file_static(logDir).getCanonicalFile().toString() + "/results/h2ologs"); + resultsDir = new File(locateFile(logDir).getCanonicalFile().toString() + "/results"); + h2oLogsDir = new File(locateFile(logDir).getCanonicalFile().toString() + "/results/h2ologs"); } catch (IOException e) { System.out.println("Couldn't create directory."); e.printStackTrace(); @@ -54,7 +56,7 @@ private void accuracySuiteSetup(@org.testng.annotations.Optional("h2o-test-accur File suiteSummary; try { - suiteSummary = new File(AccuracyTestingUtil.find_test_file_static(logDir).getCanonicalFile().toString() + + suiteSummary = new File(locateFile(logDir).getCanonicalFile().toString() + "/results/accuracySuiteSummary.log"); suiteSummary.createNewFile(); summaryLog = new PrintStream(new FileOutputStream(suiteSummary, false)); @@ -83,7 +85,7 @@ private void accuracySuiteSetup(@org.testng.annotations.Optional("h2o-test-accur // Data sets this.dataSetsCSVPath = dataSetsCSVPath; - File dataSetsFile = AccuracyTestingUtil.find_test_file_static(this.dataSetsCSVPath); + File dataSetsFile = locateFile(this.dataSetsCSVPath); try { dataSetsCSVRows = Files.readAllLines(dataSetsFile.toPath(), Charset.defaultCharset()); } catch (IOException e) { @@ -164,7 +166,7 @@ public void accuracyTest() { try { summaryLog.println("Reading test cases from: " + testCasesCSVPath); - File testCasesFile = AccuracyTestingUtil.find_test_file_static(this.testCasesCSVPath); + File testCasesFile = locateFile(this.testCasesCSVPath); testCaseEntries = Files.readAllLines(testCasesFile.toPath(), Charset.defaultCharset()); } catch (Exception e) { diff --git a/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java b/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java index a6f3b649d41..40ff821c496 100644 --- a/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java +++ b/h2o-test-accuracy/src/test/java/water/AccuracyTestingUtil.java @@ -1,6 +1,7 @@ package water; import java.util.UUID; +import static water.util.FileUtils.*; public class AccuracyTestingUtil extends TestUtil { private static boolean _stall_called_before = false; @@ -18,7 +19,7 @@ public static void stall_till_cloudsize(int x, String logDir) { if (H2O.getCloudSize() < x) { // Leader node, where the tests execute from. String cloudName = UUID.randomUUID().toString(); - String[] args = new String[]{"-name",cloudName,"-ice_root",find_test_file_static(logDir + "/results"). + String[] args = new String[]{"-name",cloudName,"-ice_root", locateFile(logDir + "/results"). toString()}; H2O.main(args); diff --git a/h2o-test-accuracy/src/test/java/water/DataSet.java b/h2o-test-accuracy/src/test/java/water/DataSet.java index 2eebdebd26a..460e4960fa2 100644 --- a/h2o-test-accuracy/src/test/java/water/DataSet.java +++ b/h2o-test-accuracy/src/test/java/water/DataSet.java @@ -4,6 +4,8 @@ import java.io.IOException; import water.fvec.Frame; +import water.util.FileUtils; +import static water.util.FileUtils.*; public class DataSet { @@ -63,6 +65,6 @@ private File makeDataSetFile(String uri) { String[] uriTokens = uri.trim().split("/", -1); String fileName = uriTokens[uriTokens.length - 1]; - return AccuracyTestingUtil.find_test_file_static(filePath + fileName); + return locateFile(filePath + fileName); } } From a37edc8763747978329ee636a3b785befa01f1bd Mon Sep 17 00:00:00 2001 From: Vlad Patryshev Date: Tue, 21 Feb 2017 12:43:10 -0800 Subject: [PATCH 2/2] More test updates after Pasha's suggestions. --- .../DeepLearningAutoEncoderCategoricalTest.java | 2 +- .../deeplearning/DeepLearningAutoEncoderTest.java | 4 +-- .../DeepLearningCheckpointReporting.java | 3 +- .../hex/deeplearning/DeepLearningMissingTest.java | 2 +- .../hex/deeplearning/DeepLearningProstateTest.java | 4 +-- .../DeepLearningReproducibilityTest.java | 4 +-- .../hex/deeplearning/DeepLearningScoreTest.java | 2 +- .../hex/deeplearning/DeepLearningSpiralsTest.java | 2 +- .../deepwater/DeepWaterMXNetIntegrationTest.java | 2 +- h2o-algos/src/test/java/hex/glrm/GLRMTest.java | 2 +- h2o-algos/src/test/java/hex/grep/GrepTest.java | 6 ++-- .../src/test/java/hex/tree/gbm/GBMMissingTest.java | 2 +- .../src/jmh/java/water/mojo/glm/GlmMojoBench.java | 2 +- .../water/mojo/glm/GlmMultinomialMojoBench.java | 2 +- h2o-core/src/main/java/water/fvec/NFSFileVec.java | 7 +++++ h2o-core/src/main/java/water/util/FileUtils.java | 1 + .../src/test/java/hex/ConfusionMatrixTest.java | 4 +-- h2o-core/src/test/java/water/TestUtil.java | 34 ++++++++++------------ h2o-core/src/test/java/water/fvec/FVecTest.java | 3 +- .../test/java/water/fvec/RebalanceDatasetTest.java | 4 +-- .../src/test/java/water/fvec/WordCountBigTest.java | 3 +- .../src/test/java/water/fvec/WordCountTest.java | 2 +- .../water/network/SSLSocketChannelFactoryTest.java | 4 +-- .../water/parser/ParseCompressedAndXLSTest.java | 3 +- .../test/java/water/parser/ParseFolderTestBig.java | 17 ++++++----- .../src/test/java/water/parser/ParserTest.java | 4 +-- .../src/test/java/water/parser/ParserTestARFF.java | 2 +- .../src/test/java/water/rapids/MungingTest.java | 4 +-- .../src/test/java/water/persist/PersistS3Test.java | 2 +- h2o-scala/src/test/scala/water/BasicTest.scala | 4 +-- .../src/test/scala/water/fvec/FrameOpsTest.scala | 4 +-- .../src/test/java/water/AccuracyTestingSuite.java | 2 +- 32 files changed, 72 insertions(+), 71 deletions(-) diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java index d5656ffd3c8..fbb25ca5af5 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderCategoricalTest.java @@ -22,7 +22,7 @@ public void run() { long seed = 0xDECAF; - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(PATH)); + NFSFileVec nfs = TestUtil.makeNfsFileVec(PATH); Frame train = ParseDataset.parse(Key.make("train.hex"), nfs._key); DeepLearningParameters p = new DeepLearningParameters(); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java index 1ec799f0b4d..f059e669bb4 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningAutoEncoderTest.java @@ -35,9 +35,9 @@ public void run() { try { - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(PATH)); + NFSFileVec nfs = TestUtil.makeNfsFileVec(PATH); train = ParseDataset.parse(Key.make("train.hex"), nfs._key); - NFSFileVec nfs2 = NFSFileVec.make(FileUtils.locateFile(PATH2)); + NFSFileVec nfs2 = TestUtil.makeNfsFileVec(PATH2); test = ParseDataset.parse(Key.make("test.hex"), nfs2._key); for (float sparsity_beta : new float[]{0, 0.1f}) { diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java index 0e2b8b66af0..c780476e4d0 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningCheckpointReporting.java @@ -25,8 +25,7 @@ Scope.enter(); Frame frame = null; try { - File file = FileUtils.locateFile("smalldata/logreg/prostate.csv"); - NFSFileVec trainfv = NFSFileVec.make(file); + NFSFileVec trainfv = TestUtil.makeNfsFileVec("smalldata/logreg/prostate.csv"); frame = ParseDataset.parse(Key.make(), trainfv._key); DeepLearningParameters p = new DeepLearningParameters(); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java index 21a2f1099d7..de963fef5a4 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningMissingTest.java @@ -49,7 +49,7 @@ double loss =0; try { Scope.enter(); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); + NFSFileVec nfs = NFSFileVec.make("smalldata/junit/weather.csv"); data = ParseDataset.parse(Key.make("data.hex"), nfs._key); Log.info("FrameSplitting"); // Create holdout test data on clean data (before adding missing values) diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java index 760d597ada4..989f02182a6 100755 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningProstateTest.java @@ -49,9 +49,9 @@ public void runFraction(float fraction) { for (final int resp : responses[i]) { Frame frame=null, vframe=null; try { - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(dataset)); + NFSFileVec nfs = TestUtil.makeNfsFileVec(dataset); frame = ParseDataset.parse(Key.make(), nfs._key); - NFSFileVec vnfs = NFSFileVec.make(FileUtils.locateFile(dataset)); + NFSFileVec vnfs = TestUtil.makeNfsFileVec(dataset); vframe = ParseDataset.parse(Key.make(), vnfs._key); boolean classification = !(i == 0 && resp == 2); String respname = frame.name(resp); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java index 1b8473a1d34..a7f1cc14205 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningReproducibilityTest.java @@ -20,7 +20,7 @@ @Test public void run() { - NFSFileVec ff = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); + NFSFileVec ff = TestUtil.makeNfsFileVec("smalldata/junit/weather.csv"); Frame golden = ParseDataset.parse(Key.make("golden.hex"), ff._key); DeepLearningModel mymodel = null; @@ -40,7 +40,7 @@ public void run() { double[] numbers = new double[N]; for (int repeat = 0; repeat < N; ++repeat) { try { - NFSFileVec file = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); + NFSFileVec file = TestUtil.makeNfsFileVec("smalldata/junit/weather.csv"); data = ParseDataset.parse(Key.make("data.hex"), file._key); Assert.assertTrue(TestUtil.isBitIdentical(data, golden)); //test parser consistency diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java index e02cd415562..d14f91cef8c 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningScoreTest.java @@ -28,7 +28,7 @@ @Test public void testPubDev928() { // Create rebalanced dataset Key rebalancedKey = Key.make("rebalanced"); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/logreg/prostate.csv")); + NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/logreg/prostate.csv"); Frame fr = ParseDataset.parse(Key.make(), nfs._key); RebalanceDataSet rb = new RebalanceDataSet(fr, rebalancedKey, (int)(fr.numRows()+1)); H2O.submitTask(rb); diff --git a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java index 7181cfd5338..93468c0eb43 100644 --- a/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java +++ b/h2o-algos/src/test/java/hex/deeplearning/DeepLearningSpiralsTest.java @@ -21,7 +21,7 @@ @Test public void run() { Scope.enter(); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/two_spiral.csv")); + NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/junit/two_spiral.csv"); Frame frame = ParseDataset.parse(Key.make(), nfs._key); Log.info(frame); int resp = frame.names().length-1; diff --git a/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java b/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java index 4934918c369..e91483f25b5 100644 --- a/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java +++ b/h2o-algos/src/test/java/hex/deepwater/DeepWaterMXNetIntegrationTest.java @@ -68,7 +68,7 @@ public void inceptionPredictionMX() throws IOException { float[] mean = backend.loadMeanImage(_model, extractFile(path, "mean_224.nd")); // Turn the image into a vector of the correct size - File imgFile = FileUtils.locateFile("smalldata/deepwater/imagenet/test2.jpg"); + File imgFile = FileUtils.getFile("smalldata/deepwater/imagenet/test2.jpg"); BufferedImage img = ImageIO.read(imgFile); BufferedImage scaledImg = new BufferedImage(w, h, img.getType()); Graphics2D g2d = scaledImg.createGraphics(); diff --git a/h2o-algos/src/test/java/hex/glrm/GLRMTest.java b/h2o-algos/src/test/java/hex/glrm/GLRMTest.java index fc5532e56fc..eb14f344bae 100644 --- a/h2o-algos/src/test/java/hex/glrm/GLRMTest.java +++ b/h2o-algos/src/test/java/hex/glrm/GLRMTest.java @@ -101,7 +101,7 @@ public static void checkLossbyCol(GLRMParameters parms, GLRMModel model) { Frame train; InputStream is; try { - is = new FileInputStream(FileUtils.locateFile("bigdata/laptop/census/ACS_13_5YR_DP02_cleaned.zip")); + is = new FileInputStream(FileUtils.getFile("bigdata/laptop/census/ACS_13_5YR_DP02_cleaned.zip")); UploadFileVec.ReadPutStats stats = new UploadFileVec.ReadPutStats(); UploadFileVec.readPut("train",is,stats); } catch (Exception e) { diff --git a/h2o-algos/src/test/java/hex/grep/GrepTest.java b/h2o-algos/src/test/java/hex/grep/GrepTest.java index 08a33b68ba8..154aa5abf12 100644 --- a/h2o-algos/src/test/java/hex/grep/GrepTest.java +++ b/h2o-algos/src/test/java/hex/grep/GrepTest.java @@ -23,10 +23,8 @@ //TODO: fix with original regex //String regex = "Iris-versicolor"; String regex = "ver..c\\wl[ob]r"; - File f = FileUtils.locateFile("smalldata/iris/iris_wheader.csv"); - //String regex = "(?:(\\w)\\1){5}"; - //File f = new File("bigdata/text8.txt"); - NFSFileVec nfs = NFSFileVec.make(f); + NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/iris/iris_wheader.csv"); + DKV.put(fr = new Frame(Key.make(), new String[]{"text"}, new Vec[]{nfs})); // long now = System.nanoTime(); GrepModel.GrepParameters parms = new GrepModel.GrepParameters(); diff --git a/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java b/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java index b41b51dfda6..4609008507d 100644 --- a/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java +++ b/h2o-algos/src/test/java/hex/tree/gbm/GBMMissingTest.java @@ -41,7 +41,7 @@ double err=0; try { Scope.enter(); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("smalldata/junit/weather.csv")); + NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/junit/weather.csv"); data = ParseDataset.parse(Key.make("data.hex"), nfs._key); Log.info("FrameSplitting"); // Create holdout test data on clean data (before adding missing values) diff --git a/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java b/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java index bdad77af5d8..2e0704ae9be 100644 --- a/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java +++ b/h2o-core/src/jmh/java/water/mojo/glm/GlmMojoBench.java @@ -41,7 +41,7 @@ @Setup public void setup() throws IOException { - File f = locateFile("smalldata/airlines/allyears2k.zip"); + File f = getFile("smalldata/airlines/allyears2k.zip"); mojo = (GlmMojoModel) loadMojo("airlines"); diff --git a/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java b/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java index b310e5cf51e..f6f01984111 100644 --- a/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java +++ b/h2o-core/src/jmh/java/water/mojo/glm/GlmMultinomialMojoBench.java @@ -42,7 +42,7 @@ @Setup public void setup() throws IOException { - File f = locateFile("smalldata/flow_examples/mnist/test.csv.gz"); + File f = getFile("smalldata/flow_examples/mnist/test.csv.gz"); mojo = (GlmMultinomialMojoModel) loadMojo("mnist"); diff --git a/h2o-core/src/main/java/water/fvec/NFSFileVec.java b/h2o-core/src/main/java/water/fvec/NFSFileVec.java index ed1f0a0d20a..078ed0e8b46 100644 --- a/h2o-core/src/main/java/water/fvec/NFSFileVec.java +++ b/h2o-core/src/main/java/water/fvec/NFSFileVec.java @@ -1,9 +1,11 @@ package water.fvec; import java.io.File; +import java.io.IOException; import water.*; import water.persist.PersistNFS; +import water.util.FileUtils; /** A NFS distributed file-backed Vector *

@@ -29,6 +31,11 @@ public static NFSFileVec make(File f) { return nfs; } + public static NFSFileVec make(String fname) throws IOException { + File f = FileUtils.getFile(fname); + return NFSFileVec.make(f); + } + /** Make a new NFSFileVec key which holds the filename implicitly. This name * is used by the Chunks to load data on-demand. * @return A NFSFileVec mapped to this file. */ diff --git a/h2o-core/src/main/java/water/util/FileUtils.java b/h2o-core/src/main/java/water/util/FileUtils.java index 9cf4387eb62..809b338a9c0 100644 --- a/h2o-core/src/main/java/water/util/FileUtils.java +++ b/h2o-core/src/main/java/water/util/FileUtils.java @@ -2,6 +2,7 @@ import org.eclipse.jetty.io.EofException; import water.Key; +import water.fvec.NFSFileVec; import java.io.*; import java.net.URI; diff --git a/h2o-core/src/test/java/hex/ConfusionMatrixTest.java b/h2o-core/src/test/java/hex/ConfusionMatrixTest.java index a3162c2e256..6be71d559c5 100644 --- a/h2o-core/src/test/java/hex/ConfusionMatrixTest.java +++ b/h2o-core/src/test/java/hex/ConfusionMatrixTest.java @@ -197,8 +197,8 @@ public void testDifferentDomainsNumericVectors() { private void simpleCMTest(String f1, String f2, String[] expectedActualDomain, String[] expectedPredictDomain, String[] expectedDomain, double[][] expectedCM, boolean debug) { try { - Frame v1 = parseFrame(Key.make("v1.hex"), FileUtils.locateFile(f1)); - Frame v2 = parseFrame(Key.make("v2.hex"), FileUtils.locateFile(f2)); + Frame v1 = parseFrame(Key.make("v1.hex"), FileUtils.getFile(f1)); + Frame v2 = parseFrame(Key.make("v2.hex"), FileUtils.getFile(f2)); if (!v1.isCompatible(v2)) { Frame old = null; v2 = new Frame(v1.makeCompatible(old = v2)); diff --git a/h2o-core/src/test/java/water/TestUtil.java b/h2o-core/src/test/java/water/TestUtil.java index 6d7ddc3ed04..7ef597f581e 100644 --- a/h2o-core/src/test/java/water/TestUtil.java +++ b/h2o-core/src/test/java/water/TestUtil.java @@ -211,14 +211,6 @@ public static boolean isBitIdentical(Frame fr1, Frame fr2) { return isIdenticalUpToRelTolerance(fr1,fr2,0); } - public static void checkFile(String name, File file) { - try { - FileUtils.checkFile(file, name); - } catch (IOException ioe) { - fail(ioe.getMessage()); - } - } - static File[] contentsOf(String name, File folder) { try { return FileUtils.contentsOf(folder, name); @@ -231,22 +223,26 @@ public static void checkFile(String name, File file) { /** Find & parse a CSV file. NPE if file not found. * @param fname Test filename * @return Frame or NPE */ - public static Frame parse_test_file( String fname ) { return parse_test_file(Key.make(),fname); } + public static Frame parse_test_file( String fname ) { + return parse_test_file(Key.make(),fname); + } + + public static NFSFileVec makeNfsFileVec(String fname) { + try { + return NFSFileVec.make(fname); + } catch (IOException ioe) { + fail(ioe.getMessage()); + return null; + } + } + public static Frame parse_test_file( Key outputKey, String fname) { NFSFileVec nfs = makeNfsFileVec(fname); return ParseDataset.parse(outputKey, nfs._key); } - static NFSFileVec makeNfsFileVec(String fname) { - File f = FileUtils.locateFile(fname); - checkFile(fname, f); - return NFSFileVec.make(f); - } - - protected Frame parse_test_file( Key outputKey, String fname , boolean guessSetup) { - File f = FileUtils.locateFile(fname); - checkFile(fname, f); - NFSFileVec nfs = NFSFileVec.make(f); + protected Frame parse_test_file( Key outputKey, String fname, boolean guessSetup) { + NFSFileVec nfs = makeNfsFileVec(fname); return ParseDataset.parse(outputKey, new Key[]{nfs._key}, true, ParseSetup.guessSetup(new Key[]{nfs._key},false,1)); } diff --git a/h2o-core/src/test/java/water/fvec/FVecTest.java b/h2o-core/src/test/java/water/fvec/FVecTest.java index 64a97ccffc9..9b5cd516194 100644 --- a/h2o-core/src/test/java/water/fvec/FVecTest.java +++ b/h2o-core/src/test/java/water/fvec/FVecTest.java @@ -108,8 +108,7 @@ public static Key makeByteVec(Key k, String... data) { // Test making a appendable vector from a plain vector @Test public void testNewVec() { // Make and insert a File8Vec to the global store - File file = FileUtils.locateFile("./smalldata/junit/cars.csv"); - NFSFileVec nfs = NFSFileVec.make(file); + NFSFileVec nfs = TestUtil.makeNfsFileVec("./smalldata/junit/cars.csv"); Vec res = new TestNewVec().doAll(new byte[]{Vec.T_NUM},nfs).outputFrame(new String[]{"v"},new String[][]{null}).anyVec(); assertEquals(nfs.at8(0)+1,res.at8(0)); assertEquals(nfs.at8(1)+1,res.at8(1)); diff --git a/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java b/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java index f247849bdd2..a03af0a021c 100644 --- a/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java +++ b/h2o-core/src/test/java/water/fvec/RebalanceDatasetTest.java @@ -13,8 +13,8 @@ @BeforeClass public static void setup() { stall_till_cloudsize(1); } @Test public void testProstate(){ NFSFileVec[] nfs = new NFSFileVec[]{ - NFSFileVec.make(FileUtils.locateFile("smalldata/logreg/prostate.csv")), - NFSFileVec.make(FileUtils.locateFile("smalldata/covtype/covtype.20k.data"))}; + TestUtil.makeNfsFileVec("smalldata/logreg/prostate.csv"), + TestUtil.makeNfsFileVec("smalldata/covtype/covtype.20k.data")}; //NFSFileVec.make(find_test_file("bigdata/laptop/usecases/cup98VAL_z.csv"))}; for (NFSFileVec fv : nfs) { Frame fr = ParseDataset.parse(Key.make(), fv._key); diff --git a/h2o-core/src/test/java/water/fvec/WordCountBigTest.java b/h2o-core/src/test/java/water/fvec/WordCountBigTest.java index ff2b173e86c..02fabe1927d 100644 --- a/h2o-core/src/test/java/water/fvec/WordCountBigTest.java +++ b/h2o-core/src/test/java/water/fvec/WordCountBigTest.java @@ -9,8 +9,7 @@ @Test public void testWordCountWiki() throws IOException { String best = "/home/0xdiag/datasets/wiki.xml"; File file = FileUtils.locateFile(best); - if( file==null ) file = FileUtils.locateFile("../datasets/Wiki_20130805.xml"); - if( file==null ) throw new FileNotFoundException(best); + if( file==null ) file = FileUtils.getFile("../datasets/Wiki_20130805.xml"); doWordCount(file); } diff --git a/h2o-core/src/test/java/water/fvec/WordCountTest.java b/h2o-core/src/test/java/water/fvec/WordCountTest.java index bd177e3d29e..74ccffbe658 100644 --- a/h2o-core/src/test/java/water/fvec/WordCountTest.java +++ b/h2o-core/src/test/java/water/fvec/WordCountTest.java @@ -15,7 +15,7 @@ // ========================================================================== @Test public void testWordCount() throws IOException { - File file = FileUtils.locateFile("./smalldata/junit/cars.csv"); + File file = FileUtils.getFile("./smalldata/junit/cars.csv"); doWordCount(file); } diff --git a/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java b/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java index 063f5f1bd80..75f39a5c6fe 100644 --- a/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java +++ b/h2o-core/src/test/java/water/network/SSLSocketChannelFactoryTest.java @@ -24,9 +24,9 @@ public void shouldHandshake() throws IOException, SSLContextException, BrokenBarrierException, InterruptedException { SSLProperties props = new SSLProperties(); props.put("h2o_ssl_protocol", SecurityUtils.defaultTLSVersion()); - props.put("h2o_ssl_jks_internal", locateFile("src/test/resources/keystore.jks").getPath()); + props.put("h2o_ssl_jks_internal", getFile("src/test/resources/keystore.jks").getPath()); props.put("h2o_ssl_jks_password", "password"); - props.put("h2o_ssl_jts", locateFile("src/test/resources/cacerts.jks").getPath()); + props.put("h2o_ssl_jts", getFile("src/test/resources/cacerts.jks").getPath()); props.put("h2o_ssl_jts_password", "password"); final SSLSocketChannelFactory factory = new SSLSocketChannelFactory(props); diff --git a/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java b/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java index 1ca58cdd5d1..725dd86e06f 100644 --- a/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java +++ b/h2o-core/src/test/java/water/parser/ParseCompressedAndXLSTest.java @@ -51,8 +51,7 @@ @Test public void testXLSBadArgs(){ Frame k1 = null; try { - File f = FileUtils.locateFile("smalldata/airlines/AirlinesTest.csv.zip"); - NFSFileVec nfs = NFSFileVec.make(f); + NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/airlines/AirlinesTest.csv.zip"); byte[] ctypes = new byte[12]; for(int i=0; i < 12; i++) ctypes[i] = Vec.T_NUM; ParseSetup setup = new ParseSetup(XLS_INFO, diff --git a/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java b/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java index 7a6cf577ba6..41cc3b154ae 100644 --- a/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java +++ b/h2o-core/src/test/java/water/parser/ParseFolderTestBig.java @@ -1,6 +1,7 @@ package water.parser; import java.io.File; +import java.io.IOException; import java.util.Arrays; import org.junit.*; import water.Job; @@ -54,18 +55,20 @@ public void testBIGSVM() { String fname = "bigdata/cust_K/1m.svm"; Frame k1 = null; try { - File f = FileUtils.locateFile(fname); - assert f != null && f.exists():" file not found: " + fname; + File f = FileUtils.getFile(fname); NFSFileVec nfs = NFSFileVec.make(f); - Job job = ParseDataset.parse(Key.make("BIGSVM.hex"),new Key[]{nfs._key},true,ParseSetup.guessSetup(new Key[]{nfs._key}, false, ParseSetup.GUESS_HEADER),false)._job; - while( job.progress() < 1.0 ) { - System.out.print(((int)(job.progress()*1000.0))/10.0 + "% "); - try { Thread.sleep(1000); } catch (InterruptedException ignore) { /*comment to disable ideaJ warning*/} + Job job = ParseDataset.parse(Key.make("BIGSVM.hex"), new Key[]{nfs._key}, true, ParseSetup.guessSetup(new Key[]{nfs._key}, false, ParseSetup.GUESS_HEADER), false)._job; + while (job.progress() < 1.0) { + System.out.print(((int) (job.progress() * 1000.0)) / 10.0 + "% "); + try { + Thread.sleep(1000); + } catch (InterruptedException ignore) { /*comment to disable ideaJ warning*/} } System.out.println(); k1 = job.get(); System.out.println(k1.toString()); - + } catch (IOException ioe) { + Assert.fail("File not found: " + fname + " - " + ioe.getMessage()); } finally { if( k1 != null ) k1.delete(); } diff --git a/h2o-core/src/test/java/water/parser/ParserTest.java b/h2o-core/src/test/java/water/parser/ParserTest.java index 13d3a39b4f2..bdf2cdd296c 100644 --- a/h2o-core/src/test/java/water/parser/ParserTest.java +++ b/h2o-core/src/test/java/water/parser/ParserTest.java @@ -845,7 +845,7 @@ static void testParsed(Frame fr, double[][] expected, int len) { }) { try { Log.info("Trying to parse " + f); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(f)); + NFSFileVec nfs = TestUtil.makeNfsFileVec(f); Frame fr = ParseDataset.parse(Key.make(), new Key[]{nfs._key}, delete_on_done, true /*single quote*/, check_header); fr.delete(); } catch (Throwable t) { @@ -886,7 +886,7 @@ public void testPubDev2897() { @Test public void testParserRespectsSpecifiedColNum() { - Vec fv = NFSFileVec.make(FileUtils.locateFile("smalldata/jira/runit_pubdev_3590_unexpected_column.csv")); + Vec fv = TestUtil.makeNfsFileVec("smalldata/jira/runit_pubdev_3590_unexpected_column.csv"); Key fkey = Key.make("data4cols"); try { Key[] keys = new Key[]{fv._key}; diff --git a/h2o-core/src/test/java/water/parser/ParserTestARFF.java b/h2o-core/src/test/java/water/parser/ParserTestARFF.java index bd78fec5997..8e4cbe040dc 100644 --- a/h2o-core/src/test/java/water/parser/ParserTestARFF.java +++ b/h2o-core/src/test/java/water/parser/ParserTestARFF.java @@ -93,7 +93,7 @@ private void testColNames(String[] dataset, String[] exp, int len, String sep) { * @param fname Test filename * @return Frame or NPE */ private Frame parse_test_file_single_quotes( String fname ) { - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile(fname)); + NFSFileVec nfs = TestUtil.makeNfsFileVec(fname); return ParseDataset.parse(Key.make(), new Key[]{nfs._key}, true, true /*single quote*/, ParseSetup.GUESS_HEADER); } diff --git a/h2o-core/src/test/java/water/rapids/MungingTest.java b/h2o-core/src/test/java/water/rapids/MungingTest.java index fd43a8b0514..0ceea45aaf5 100644 --- a/h2o-core/src/test/java/water/rapids/MungingTest.java +++ b/h2o-core/src/test/java/water/rapids/MungingTest.java @@ -32,7 +32,7 @@ private void copyStream(OutputStream os, InputStream is, final int buffer_size) } @Ignore @Test public void run2() throws Exception { System.out.println("Running run2 ..."); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("/home/mdowle/devtestdata/step1.csv")); + NFSFileVec nfs = TestUtil.makeNfsFileVec("/home/mdowle/devtestdata/step1.csv"); Frame frame = ParseDataset.parse(Key.make(), nfs._key); // look into parse() to manip column types System.out.println("Loaded file, now calling Query ..."); // new RadixOrder(frame, true, new int[] {0,1}); // group by 0=id, 1=date and sum 3 == quantity @@ -43,7 +43,7 @@ private void copyStream(OutputStream os, InputStream is, final int buffer_size) @Ignore @Test public void run3() throws Exception { System.out.println("Running run3 ..."); - NFSFileVec nfs = NFSFileVec.make(FileUtils.locateFile("/home/mdowle/devtestdata/step1_subset.csv")); + NFSFileVec nfs = TestUtil.makeNfsFileVec("/home/mdowle/devtestdata/step1_subset.csv"); //NFSFileVec nfs = NFSFileVec.make(find_test_file("/users/arno/devtestdata/step1_subset.csv")); Frame leftFrame = ParseDataset.parse(Key.make(), nfs._key); diff --git a/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java b/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java index 2c3b54f2538..fd05c3a106f 100644 --- a/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java +++ b/h2o-persist-s3/src/test/java/water/persist/PersistS3Test.java @@ -43,7 +43,7 @@ public void testS3Import() throws Exception { int chunkSize = (int) (v.length() / 3); v.setChunkSize(fr, chunkSize); long xor = new XORTask().doAll(v)._res; - Key k2 = H2O.getPM().anyURIToKey(new URI(FileUtils.locateFile("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath())); + Key k2 = H2O.getPM().anyURIToKey(new URI(FileUtils.getFile("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath())); FileVec v2 = DKV.getGet(k2); assertEquals(v2.length(), v.length()); assertVecEquals(v, v2, 0); diff --git a/h2o-scala/src/test/scala/water/BasicTest.scala b/h2o-scala/src/test/scala/water/BasicTest.scala index b6244596e37..45c0d3ba6dd 100644 --- a/h2o-scala/src/test/scala/water/BasicTest.scala +++ b/h2o-scala/src/test/scala/water/BasicTest.scala @@ -9,8 +9,8 @@ class BasicTest extends TestUtil { @Test def testDataFrameLoadAPI(): Unit = { val filename1 = "../smalldata/iris/iris_wheader.csv" val filename2 = "../smalldata/iris/iris.csv" - val file1 = locateFile(filename1) - val file2 = locateFile(filename2) + val file1 = getFile(filename1) + val file2 = getFile(filename2) val uri1 = file1.toURI val uri2 = file2.toURI // Create frames diff --git a/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala b/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala index 1a4ea2a037c..46a2727287b 100644 --- a/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala +++ b/h2o-scala/src/test/scala/water/fvec/FrameOpsTest.scala @@ -12,7 +12,7 @@ class FrameOpsTest extends TestUtil { @Test def testParserSetup(): Unit = { - val irisFile = FileUtils.locateFile("smalldata/iris/iris.csv") + val irisFile = FileUtils.getFile("smalldata/iris/iris.csv") val f1 = new H2OFrame(irisFile) // Default setup val parserSetup = H2OFrame.defaultParserSetup() @@ -35,7 +35,7 @@ class FrameOpsTest extends TestUtil { @Test def testApplyMethod1(): Unit = { - val carsFile = FileUtils.locateFile("smalldata/junit/cars.csv") + val carsFile = FileUtils.getFile("smalldata/junit/cars.csv") val f1 = new H2OFrame(carsFile) val subframe = f1('name) try { diff --git a/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java b/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java index c05f82b4618..785f5c73587 100644 --- a/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java +++ b/h2o-test-accuracy/src/test/java/water/AccuracyTestingSuite.java @@ -166,7 +166,7 @@ public void accuracyTest() { try { summaryLog.println("Reading test cases from: " + testCasesCSVPath); - File testCasesFile = locateFile(this.testCasesCSVPath); + File testCasesFile = getFile(testCasesCSVPath); testCaseEntries = Files.readAllLines(testCasesFile.toPath(), Charset.defaultCharset()); } catch (Exception e) {