Cleanup read and write bytes in test & pom improvements

src/test/java/org/apache/datasketches/common/A_BeforeSuite.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.common;
+
+import org.testng.annotations.BeforeSuite;
+
+public class A_BeforeSuite {
+
+  @BeforeSuite(alwaysRun = true)
+  public void printTestEnvironment() {
+    System.out.println("====================================================");
+    System.out.println("TEST JDK: " + System.getProperty("java.version"));
+    System.out.println("TEST JDK HOME: " + System.getProperty("java.home"));
+    System.out.println("=====================================================");
+  }
+}

src/test/java/org/apache/datasketches/common/TestUtil.java

@@ -19,248 +19,122 @@
 
 package org.apache.datasketches.common;
 
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
 import java.util.Objects;
 
 /**
  * Utilities common to testing
  */
 public final class TestUtil  {
 
-  private static final String userDir = System.getProperty("user.dir");
-
   /**
    * TestNG group constants
    */
   public static final String GENERATE_JAVA_FILES = "generate_java_files";
   public static final String CHECK_CPP_FILES = "check_cpp_files";
   public static final String CHECK_GO_FILES = "check_go_files";
+  public static final String CHECK_RUST_FILES = "check_rust_files";
   public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files";
 
   /**
-   * The full target Path for Java serialized sketches to be tested by other languages.
+   * The project relative Path for Java serialized sketches to be tested by other languages.
    */
-  public static final Path javaPath = createPath("serialization_test_data/java_generated_files");
+  public static final Path javaPath = Path.of(".", "serialization_test_data", "java_generated_files");
 
   /**
-   * The full target Path for C++ serialized sketches to be tested by Java.
+   * The project relative Path for C++ serialized sketches to be tested by Java.
    */
-  public static final Path cppPath = createPath("serialization_test_data/cpp_generated_files");
+  public static final Path cppPath = Path.of(".", "serialization_test_data", "cpp_generated_files");
 
   /**
-   * The full target Path for Go serialized sketches to be tested by Java.
+   * The project relative Path for Go serialized sketches to be tested by Java.
    */
-  public static final Path goPath = createPath("serialization_test_data/go_generated_files");
-
-  private static Path createPath(final String projectLocalDir) {
-    try {
-      return Files.createDirectories(Paths.get(userDir, projectLocalDir));
-    } catch (IOException e) { throw new SketchesArgumentException(e.getCause().toString()); }
-  }
-
-  //Get Resources
-
-  private static final int BUF_SIZE = 1 << 13;
+  public static final Path goPath = Path.of(".", "serialization_test_data", "go_generated_files");
 
   /**
-   * Gets the file defined by the given resource file's shortFileName.
-   * @param shortFileName the last name in the pathname's name sequence.
-   * @return the file defined by the given resource file's shortFileName.
+   * The project relative Path for Rust serialized sketches to be tested by Java.
    */
-  public static File getResourceFile(final String shortFileName) {
-    Objects.requireNonNull(shortFileName, "input parameter 'String shortFileName' cannot be null.");
-    final String slashName = (shortFileName.charAt(0) == '/') ? shortFileName : '/' + shortFileName;
-    final URL url = TestUtil.class.getResource(slashName);
-    Objects.requireNonNull(url, "resource " + slashName + " returns null URL.");
-    File file;
-    file = createTempFile(slashName);
-    if (url.getProtocol().equals("jar")) { //definitely a jar
-      try (final InputStream input = TestUtil.class.getResourceAsStream(slashName);
-        final OutputStream out = new FileOutputStream(file)) {
-        Objects.requireNonNull(input, "InputStream  is null.");
-        int numRead = 0;
-        final byte[] buf = new byte[1024];
-        while ((numRead = input.read(buf)) != -1) { out.write(buf, 0, numRead); }
-      } catch (final IOException e ) { throw new RuntimeException(e); }
-    } else { //protocol says resource is not a jar, must be a file
-      file = new File(getResourcePath(url));
-    }
-    if (!file.setReadable(false, true)) {
-      throw new IllegalStateException("Failed to set owner only 'Readable' on file");
-    }
-    if (!file.setWritable(false, false)) {
-      throw new IllegalStateException("Failed to set everyone 'Not Writable' on file");
-    }
-    return file;
-  }
+  public static final Path rustPath = Path.of(".", "serialization_test_data", "rust_generated_files");
 
   /**
-   * Returns a byte array of the contents of the file defined by the given resource file's shortFileName.
-   * @param shortFileName the last name in the pathname's name sequence.
-   * @return a byte array of the contents of the file defined by the given resource file's shortFileName.
-   * @throws IllegalArgumentException if resource cannot be read.
+   * The project relative Path for /src/test/resources
    */
-  public static byte[] getResourceBytes(final String shortFileName) {
-    Objects.requireNonNull(shortFileName, "input parameter 'String shortFileName' cannot be null.");
-    final String slashName = (shortFileName.charAt(0) == '/') ? shortFileName : '/' + shortFileName;
-    final URL url = TestUtil.class.getResource(slashName);
-    Objects.requireNonNull(url, "resource " + slashName + " returns null URL.");
-    final byte[] out;
-    if (url.getProtocol().equals("jar")) { //definitely a jar
-      try (final InputStream input = TestUtil.class.getResourceAsStream(slashName)) {
-        out = readAllBytesFromInputStream(input);
-      } catch (final IOException e) { throw new RuntimeException(e); }
-    } else { //protocol says resource is not a jar, must be a file
-      try {
-        out = Files.readAllBytes(Paths.get(getResourcePath(url)));
-      } catch (final IOException e) { throw new RuntimeException(e); }
-    }
-    return out;
-  }
+  public static final Path resPath = Path.of(".","src","test","resources");
 
-  /**
-   * Note: This is only needed in Java 8 as it is part of Java 9+.
-   * Read all bytes from the given <i>InputStream</i>.
-   * This is limited to streams that are no longer than the maximum allocatable byte array determined by the VM.
-   * This may be a little smaller than <i>Integer.MAX_VALUE</i>.
-   * @param in the Input Stream
-   * @return byte array
-   */
-  public static byte[] readAllBytesFromInputStream(final InputStream in) {
-    return readBytesFromInputStream(Integer.MAX_VALUE, in);
-  }
+  public enum Existence { MUST_EXIST, WARNING }
 
   /**
-   * Note: This is only needed in Java 8 as is part of Java 9+.
-   * Read <i>numBytesToRead</i> bytes from an input stream into a single byte array.
-   * This is limited to streams that are no longer than the maximum allocatable byte array determined by the VM.
-   * This may be a little smaller than <i>Integer.MAX_VALUE</i>.
-   * @param numBytesToRead number of bytes to read
-   * @param in the InputStream
-   * @return the filled byte array from the input stream
-   * @throws IllegalArgumentException if array size grows larger than what can be safely allocated by some VMs.
-
+   * Gets all the bytes of a file as a byte array.
+   * If the file is missing, this either throws an exception or writes a warning message to the console
+   * based on the state of the optional {@link #Existence Existence}.
+   * @param basePath the base directory path where the file is located
+   * @param fileName the simple file name of the file
+   * @param option an optional parameter. If option == Existence.MUST_EXIST and the file does not exist an exception will be thrown.
+   * If option == Existence.WARNING, or not given, and the file does not exist, it writes a warning message
+   * to {@link System.err.out System.err.out}.
+   * If option has more than one argument an exception will be thrown.
+   * @return a byte array. It may be empty.
+   * @throws RuntimeException for IO errors, or if resolved path is not a file or not readable or optionally not found.
    */
-  public static byte[] readBytesFromInputStream(final int numBytesToRead, final InputStream in) {
-    if (numBytesToRead < 0) { throw new IllegalArgumentException("numBytesToRead must be positive or zero."); }
-
-    List<byte[]> buffers = null;
-    byte[] result = null;
-    int totalBytesRead = 0;
-    int remaining = numBytesToRead;
-    int chunkCnt;
-    do {
-        final byte[] partialBuffer = new byte[Math.min(remaining, BUF_SIZE)];
-        int numRead = 0;
-
-        try {
-          // reads input stream in chunks of partial buffers, stops at EOF or when remaining is zero.
-          while ((chunkCnt =
-                in.read(partialBuffer, numRead, Math.min(partialBuffer.length - numRead, remaining))) > 0) {
-              numRead += chunkCnt;
-              remaining -= chunkCnt;
-          }
-        } catch (final IOException e) { throw new RuntimeException(e); }
-
-        if (numRead > 0) {
-            if (Integer.MAX_VALUE - Long.BYTES - totalBytesRead < numRead) {
-              throw new IllegalArgumentException(
-                  "Input stream is larger than what can be safely allocated as a byte[] in some VMs."); }
-            totalBytesRead += numRead;
-            if (result == null) {
-                result = partialBuffer;
-            } else {
-                if (buffers == null) {
-                    buffers = new ArrayList<>();
-                    buffers.add(result);
-                }
-                buffers.add(partialBuffer);
-            }
-        }
-    } while (chunkCnt >= 0 && remaining > 0);
-
-    final byte[] out;
-    if (buffers == null) {
-        if (result == null) {
-          out = new byte[0];
-        } else {
-          out = result.length == totalBytesRead ? result : Arrays.copyOf(result, totalBytesRead);
-        }
-        return out;
+  public static byte[] getFileBytes(final Path basePath, final String fileName,  Existence... option) {
+    Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null.");
+    Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null.");
+    if (option.length > 1) { throw new IllegalArgumentException("Existence option has a maximum of one argument"); }
+    Existence status = (option.length == 1) ? option[0] : Existence.WARNING;
+
+    Path path = Path.of(basePath.toString(), fileName);
+    Path absPath = path.toAbsolutePath(); //for error output
+    if (Files.notExists(path)) {
+      if (status == Existence.MUST_EXIST) {
+        throw new RuntimeException("File disappeared or not found: " + absPath);
+      } else {
+        System.err.println("WARNING: File disappeared or not found: " + absPath);
+        return new byte[0];
+      }
     }
-
-    result = new byte[totalBytesRead];
-    int offset = 0;
-    remaining = totalBytesRead;
-    for (byte[] b : buffers) {
-        final int count = Math.min(b.length, remaining);
-        System.arraycopy(b, 0, result, offset, count);
-        offset += count;
-        remaining -= count;
+    if (!Files.isRegularFile(path) || !Files.isReadable(path)) {
+      throw new RuntimeException("Path is not a regular file or not readable: " + absPath);
     }
-    return result;
-  }
-
-  private static String getResourcePath(final URL url) { //must not be null
     try {
-      final URI uri = url.toURI();
-      //decodes any special characters
-      final String path = uri.isAbsolute() ? Paths.get(uri).toAbsolutePath().toString() : uri.getPath();
-      return path;
-    } catch (final URISyntaxException e) {
-      throw new IllegalArgumentException("Cannot find resource: " + url.toString() + Util.LS + e);
+      byte[] bytes = Files.readAllBytes(path);
+      return bytes;
+    } catch (IOException e) {
+        throw new RuntimeException("System IO Error reading file: " + absPath + " " + e);
     }
   }
 
   /**
-   * Create an empty temporary file.
-   * On a Mac these files are stored at the system variable $TMPDIR.  They should be cleared on a reboot.
-   * @param shortFileName the name before prefixes and suffixes are added here and by the OS.
-   * The final extension will be the current extension. The prefix "temp_" is added here.
-   * @return a temp file,which will be eventually deleted by the OS
+   * Puts all the bytes of the given byte array to a file with the given fileName.
+   * This assumes that the base directory path is {@link #javaPath javaPath}.
+   * @param fileName the name of the target file
+   * @param bytes the given byte array
    */
-  private static File createTempFile(final String shortFileName) {
-    //remove any leading slash
-    final String resName = (shortFileName.charAt(0) == '/') ? shortFileName.substring(1) : shortFileName;
-    final String suffix;
-    final String name;
-    final int  lastIdx = resName.length() - 1;
-    final int lastIdxOfDot = resName.lastIndexOf('.');
-    if (lastIdxOfDot == -1) {
-      suffix = ".tmp";
-      name = resName;
-    } else if (lastIdxOfDot == lastIdx) {
-      suffix = ".tmp";
-      name = resName.substring(0, lastIdxOfDot);
-    } else { //has a real suffix
-      suffix = resName.substring(lastIdxOfDot);
-      name = resName.substring(0, lastIdxOfDot);
-    }
-    final File file;
-    try {
-      file = File.createTempFile("temp_" + name, suffix);
-      if (!file.setReadable(false, true)) {
-        throw new IllegalStateException("Failed to set only owner 'Readable' on file");
-      }
-      if (!file.setWritable(false, true)) {
-        throw new IllegalStateException("Failed to set only owner 'Writable' on file");
-      }
-
-    } catch (final IOException e) { throw new RuntimeException(e); }
-    return file;
+  public static void putBytesToJavaPath(final String fileName, final byte[] bytes) {
+    putBytesToFile(javaPath, fileName, bytes);
   }
 
+  /**
+   * Puts all the bytes of the given byte array to a basePath file with the given fileName.
+   * If the file exists it will be overwritten.
+   * @param basePath the directory path for the given fileName
+   * @param fileName the name of the target file
+   * @param bytes the given byte array
+   * @throws RuntimeException for IO errors,
+   */
+  public static void putBytesToFile(final Path basePath, final String fileName, final byte[] bytes) {
+    Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null.");
+    Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null.");
+    Objects.requireNonNull(bytes, "input parameter 'byte[] bytes' cannot be null.");
+    Path filePath = null;
+    try {
+      Files.createDirectories(basePath); //create the directory if it doesn't exist.
+      filePath = basePath.resolve(fileName);
+      Files.write(filePath, bytes);
+    } catch (IOException e) {
+      final String filePathDesc = String.valueOf(filePath);
+      throw new RuntimeException("System IO Error writing file: " + filePathDesc + " " + e);
+    }
+  }
 }