Merge pull request #56 from slonopotamus/upload-fixes

Fixed to object upload

Author: slonopotamus

README.md

@@ -54,7 +54,10 @@ See https://github.com/bozaro/git-lfs-java/blob/master/gitlfs-server/src/test/ja
 
 ## Changes
 
-Version 0.14.0 (Unreleased)
+Version 0.14.0
+
+ * Add LFS object verification server code
+ * Fix broken handling of already existing LFS object 
 
 Version 0.13.3
 

gitlfs-client/src/main/java/ru/bozaro/gitlfs/client/Client.java

@@ -90,6 +90,56 @@ public ObjectRes getMeta(@NotNull final String hash) throws IOException {
     ), Operation.Download);
   }
 
+  /**
+   * Upload object with specified hash and size.
+   *
+   * @param streamProvider Object stream provider.
+   * @param hash           Object hash.
+   * @param size           Object size.
+   * @return Return true is object is uploaded successfully and false if object is already uploaded.
+   * @throws IOException On some errors.
+   */
+  public boolean putObject(@NotNull final StreamProvider streamProvider, @NotNull final String hash, final long size) throws IOException {
+    return putObject(streamProvider, new Meta(hash, size));
+  }
+
+  /**
+   * Upload object with specified hash and size.
+   *
+   * @param streamProvider Object stream provider.
+   * @param meta           Object metadata.
+   * @return Return true is object is uploaded successfully and false if object is already uploaded.
+   * @throws IOException On some errors.
+   */
+  public boolean putObject(@NotNull final StreamProvider streamProvider, @NotNull final Meta meta) throws IOException {
+    return doWork(auth -> {
+      final ObjectRes links = doRequest(auth, new MetaPost(meta), AuthHelper.join(auth.getHref(), PATH_OBJECTS));
+      return links != null && putObject(streamProvider, meta, links);
+    }, Operation.Upload);
+  }
+
+  protected <T> T doWork(@NotNull Work<T> work, @NotNull Operation operation) throws IOException {
+    Link auth = authProvider.getAuth(operation);
+    int authCount = 0;
+    while (true) {
+      try {
+        return work.exec(auth);
+      } catch (UnauthorizedException | ForbiddenException e) {
+        if (authCount >= MAX_AUTH_COUNT) {
+          throw e;
+        }
+        authCount++;
+        // Get new authentication data.
+        authProvider.invalidateAuth(operation, auth);
+        final Link newAuth = authProvider.getAuth(operation);
+        if (newAuth.getHeader().equals(auth.getHeader()) && newAuth.getHref().equals(auth.getHref())) {
+          throw e;
+        }
+        auth = newAuth;
+      }
+    }
+  }
+
   /**
    * Get metadata for object by hash.
    *
@@ -191,104 +241,6 @@ public boolean putObject(@NotNull final StreamProvider streamProvider) throws IO
     return putObject(streamProvider, generateMeta(streamProvider));
   }
 
-  /**
-   * Generate object metadata.
-   *
-   * @param streamProvider Object stream provider.
-   * @return Return object metadata.
-   * @throws IOException On some errors.
-   */
-  public static Meta generateMeta(@NotNull final StreamProvider streamProvider) throws IOException {
-    final MessageDigest digest = sha256();
-    final byte[] buffer = new byte[0x10000];
-    long size = 0;
-    try (InputStream stream = streamProvider.getStream()) {
-      while (true) {
-        int read = stream.read(buffer);
-        if (read <= 0) break;
-        digest.update(buffer, 0, read);
-        size += read;
-      }
-    }
-    return new Meta(new String(Hex.encodeHex(digest.digest())), size);
-  }
-
-  /**
-   * Upload object with specified hash and size.
-   *
-   * @param streamProvider Object stream provider.
-   * @param hash           Object hash.
-   * @param size           Object size.
-   * @return Return true is object is uploaded successfully and false if object is already uploaded.
-   * @throws IOException On some errors.
-   */
-  public boolean putObject(@NotNull final StreamProvider streamProvider, @NotNull final String hash, final long size) throws IOException {
-    return putObject(streamProvider, new Meta(hash, size));
-  }
-
-  /**
-   * Upload object with specified hash and size.
-   *
-   * @param streamProvider Object stream provider.
-   * @param meta           Object metadata.
-   * @return Return true is object is uploaded successfully and false if object is already uploaded.
-   * @throws IOException On some errors.
-   */
-  public boolean putObject(@NotNull final StreamProvider streamProvider, @NotNull final Meta meta) throws IOException {
-    return doWork(auth -> {
-      final ObjectRes links = doRequest(auth, new MetaPost(meta), AuthHelper.join(auth.getHref(), PATH_OBJECTS));
-      return links != null && putObject(streamProvider, meta, links);
-    }, Operation.Upload);
-  }
-
-  /**
-   * Upload object by metadata.
-   *
-   * @param links          Object links.
-   * @param streamProvider Object stream provider.
-   * @param meta           Object metadata.
-   * @return Return true is object is uploaded successfully and false if object is already uploaded.
-   * @throws IOException On some errors.
-   */
-  public boolean putObject(@NotNull final StreamProvider streamProvider, @NotNull final Meta meta, @NotNull final Links links) throws IOException {
-    if (links.getLinks().containsKey(LinkType.Download)) {
-      return false;
-    }
-    final Link uploadLink = links.getLinks().get(LinkType.Upload);
-    if (uploadLink == null) {
-      throw new IOException("Upload link not found");
-    }
-    doRequest(uploadLink, new ObjectPut(streamProvider, meta.getSize()), uploadLink.getHref());
-
-    final Link verifyLink = links.getLinks().get(LinkType.Verify);
-    if (verifyLink != null) {
-      doRequest(verifyLink, new ObjectVerify(meta), verifyLink.getHref());
-    }
-    return true;
-  }
-
-  protected <T> T doWork(@NotNull Work<T> work, @NotNull Operation operation) throws IOException {
-    Link auth = authProvider.getAuth(operation);
-    int authCount = 0;
-    while (true) {
-      try {
-        return work.exec(auth);
-      } catch (UnauthorizedException | ForbiddenException e) {
-        if (authCount >= MAX_AUTH_COUNT) {
-          throw e;
-        }
-        authCount++;
-        // Get new authentication data.
-        authProvider.invalidateAuth(operation, auth);
-        final Link newAuth = authProvider.getAuth(operation);
-        if (newAuth.getHeader().equals(auth.getHeader()) && newAuth.getHref().equals(auth.getHref())) {
-          throw e;
-        }
-        auth = newAuth;
-      }
-    }
-  }
-
   public <R> R doRequest(@Nullable Link link, @NotNull Request<R> task, @NotNull URI url) throws IOException {
     int redirectCount = 0;
     int retryCount = 0;
@@ -342,12 +294,49 @@ public <R> R doRequest(@Nullable Link link, @NotNull Request<R> task, @NotNull U
     }
   }
 
-  protected void addHeaders(@NotNull HttpUriRequest req, @Nullable Link link) {
-    if (link != null) {
-      for (Map.Entry<String, String> entry : link.getHeader().entrySet()) {
-        req.setHeader(entry.getKey(), entry.getValue());
+  /**
+   * Generate object metadata.
+   *
+   * @param streamProvider Object stream provider.
+   * @return Return object metadata.
+   * @throws IOException On some errors.
+   */
+  public static Meta generateMeta(@NotNull final StreamProvider streamProvider) throws IOException {
+    final MessageDigest digest = sha256();
+    final byte[] buffer = new byte[0x10000];
+    long size = 0;
+    try (InputStream stream = streamProvider.getStream()) {
+      while (true) {
+        int read = stream.read(buffer);
+        if (read <= 0) break;
+        digest.update(buffer, 0, read);
+        size += read;
       }
     }
+    return new Meta(new String(Hex.encodeHex(digest.digest())), size);
+  }
+
+  /**
+   * Upload object by metadata.
+   *
+   * @param links          Object links.
+   * @param streamProvider Object stream provider.
+   * @param meta           Object metadata.
+   * @return Return true is object is uploaded successfully and false if object is already uploaded.
+   * @throws IOException On some errors.
+   */
+  public boolean putObject(@NotNull final StreamProvider streamProvider, @NotNull final Meta meta, @NotNull final Links links) throws IOException {
+    final Link uploadLink = links.getLinks().get(LinkType.Upload);
+    if (uploadLink == null)
+      return false;
+
+    doRequest(uploadLink, new ObjectPut(streamProvider, meta.getSize()), uploadLink.getHref());
+
+    final Link verifyLink = links.getLinks().get(LinkType.Verify);
+    if (verifyLink != null)
+      doRequest(verifyLink, new ObjectVerify(meta), verifyLink.getHref());
+
+    return true;
   }
 
   protected static MessageDigest sha256() {
@@ -358,6 +347,14 @@ protected static MessageDigest sha256() {
     }
   }
 
+  protected void addHeaders(@NotNull HttpUriRequest req, @Nullable Link link) {
+    if (link != null) {
+      for (Map.Entry<String, String> entry : link.getHeader().entrySet()) {
+        req.setHeader(entry.getKey(), entry.getValue());
+      }
+    }
+  }
+
   @NotNull
   public Lock lock(@NotNull String path, @Nullable Ref ref) throws IOException, LockConflictException {
     final LockCreate.Res res = doWork(auth -> doRequest(

gitlfs-common/src/main/java/ru/bozaro/gitlfs/common/data/Meta.java

@@ -4,6 +4,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.jetbrains.annotations.NotNull;
 
+import java.util.Objects;
+
 /**
  * LFS object ___location.
  *
@@ -34,4 +36,18 @@ public String getOid() {
   public long getSize() {
     return size;
   }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(oid, size);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof Meta))
+      return false;
+
+    final Meta other = (Meta) o;
+    return size == other.size && oid.equals(other.oid);
+  }
 }

gitlfs-server/src/main/java/ru/bozaro/gitlfs/server/ContentServlet.java

@@ -2,6 +2,7 @@
 
 import org.jetbrains.annotations.NotNull;
 import ru.bozaro.gitlfs.common.Constants;
+import ru.bozaro.gitlfs.common.JsonHelper;
 import ru.bozaro.gitlfs.common.data.Meta;
 import ru.bozaro.gitlfs.common.io.InputStreamValidator;
 import ru.bozaro.gitlfs.server.internal.ObjectResponse;
@@ -15,6 +16,9 @@
 import java.io.InputStream;
 import java.util.regex.Pattern;
 
+import static ru.bozaro.gitlfs.server.PointerServlet.checkMimeTypes;
+import static ru.bozaro.gitlfs.server.PointerServlet.sendError;
+
 /**
  * Servlet for content storage.
  *
@@ -38,12 +42,35 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se
         return;
       }
     } catch (ServerError e) {
-      PointerServlet.sendError(resp, e);
+      sendError(resp, e);
       return;
     }
     super.doGet(req, resp);
   }
 
+  @Override
+  protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+    try {
+      checkMimeTypes(req);
+
+      if ((req.getPathInfo() != null) && PATTERN_OID.matcher(req.getPathInfo()).matches())
+        processObjectVerify(req, req.getPathInfo().substring(1)).write(resp);
+    } catch (ServerError e) {
+      sendError(resp, e);
+    }
+  }
+
+  @NotNull
+  private ResponseWriter processObjectVerify(@NotNull HttpServletRequest req, @NotNull String oid) throws IOException, ServerError {
+    manager.checkUploadAccess(req);
+    final Meta expectedMeta = JsonHelper.mapper.readValue(req.getInputStream(), Meta.class);
+    final Meta actualMeta = manager.getMetadata(oid);
+    if (!expectedMeta.equals(actualMeta))
+      throw new ServerError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, String.format("LFS verification failure: server=%s client=%s", expectedMeta, actualMeta), null);
+
+    return response -> response.setStatus(HttpServletResponse.SC_OK);
+  }
+
   @Override
   protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
     try {
@@ -52,7 +79,7 @@ protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws Se
         return;
       }
     } catch (ServerError e) {
-      PointerServlet.sendError(resp, e);
+      sendError(resp, e);
       return;
     }
     super.doPut(req, resp);
@@ -70,22 +97,19 @@ private ResponseWriter processPut(@NotNull HttpServletRequest req, @NotNull Stri
   private ResponseWriter processGet(@NotNull HttpServletRequest req, @NotNull String oid) throws ServerError, IOException {
     final ContentManager.Downloader downloader = manager.checkDownloadAccess(req);
     final InputStream stream = downloader.openObject(oid);
-    return new ResponseWriter() {
-      @Override
-      public void write(@NotNull HttpServletResponse response) throws IOException {
-        response.setStatus(HttpServletResponse.SC_OK);
-        response.setContentType(Constants.MIME_BINARY);
-        //noinspection TryFinallyCanBeTryWithResources
-        try {
-          byte[] buffer = new byte[0x10000];
-          while (true) {
-            final int read = stream.read(buffer);
-            if (read < 0) break;
-            response.getOutputStream().write(buffer, 0, read);
-          }
-        } finally {
-          stream.close();
+    return response -> {
+      response.setStatus(HttpServletResponse.SC_OK);
+      response.setContentType(Constants.MIME_BINARY);
+      //noinspection TryFinallyCanBeTryWithResources
+      try {
+        byte[] buffer = new byte[0x10000];
+        while (true) {
+          final int read = stream.read(buffer);
+          if (read < 0) break;
+          response.getOutputStream().write(buffer, 0, read);
         }
+      } finally {
+        stream.close();
       }
     };
   }