Added USDA weights to MongoDB

Also created init script to pre-seed the database once.

.gitignore

@@ -1,4 +1,7 @@
 build/
 lib/
+usda/
+\#*\#
+.\#*
 interrupt
 ensime.sbt

build.xml

@@ -5,7 +5,8 @@
   <property name="web.dir" value="web/"/>
   <property name="test.dir" value="tests/"/>
   <property name="conf.dir" value="conf/dev/"/>
-
+  <property name="script.dir" value="scripts/"/>
+  
   <property name="app.name" value="ieat"/>  
   <property name="app.conf.dir" value="${conf.dir}/app/"/>
 
@@ -82,6 +83,12 @@
     <taskdef name="junit" classname="org.apache.tools.ant.taskdefs.optional.junit.JUnitTask" />
   </target>
 
+  <target name="init.db"
+          description="Pre-populate database with seed from USDA. (Unix Only)">
+    <!-- TODO: Port Unix scripts to batch -->
+    <exec executable="${script.dir}/import_weights.sh" />
+  </target>
+
   <target name="deploy.load" depends="ivy.resolve.ant">
     <path id="ant.lib.path">
       <fileset dir="${ant.lib}" includes="*.jar"/>

dir.md

@@ -11,9 +11,11 @@
       * Same as /conf/dev
   * lib (t) - Contains dependencies dynamically loaded by ivy
     * Each sub directory represents the dependencies for a build target.
+  * scripts - Utility scripts such as database pre-seeding go here.
   * src - Java files go here.  Mostly server code
   * tests - Testing files go here
     * Mirrors /src except contains tests for each java file
+  * usda (t) - Where files downloaded from the USDA are placed
   * web - User interface files go here.
     * views - Directory of web pages the user can see
     * js - Directory of javascript files

scripts/import_weights.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+USDA_DB_FILE="sr28asc.zip"
+USDA_DB_URL="https://www.ars.usda.gov/ARSUserFiles/80400525/Data/SR/SR28/dnload/$USDA_DB_FILE"
+WEIGHT_FILE="WEIGHT.txt"
+SCRIPT_DIR="$(dirname "$0")"
+USDA_DIR="$SCRIPT_DIR/../usda"
+DB="ieat"
+COLLECTION="usda-weight"
+
+
+if [ ! -d "$USDA_DIR" ]; then
+    if ! mkdir "$USDA_DIR"; then
+        echo 1>&2 "Failed to create directory: '$USDA_DIR'."
+        exit 1
+    fi
+fi
+if [ ! -f "$USDA_DIR/$USDA_DB_FILE" ]; then
+    if ! wget -P "$USDA_DIR" "$USDA_DB_URL"; then
+        echo 1>&2 "Error downloading USDA ASCII database."
+        exit 1
+    fi
+fi
+if ! unzip -o "$USDA_DIR/$USDA_DB_FILE" $WEIGHT_FILE -d "$USDA_DIR"; then
+    echo 1>&2 "Error extracting USDA ASCII database."
+    echo 1>&2 "File, '$USDA_DIR/$USDA_DB_FILE', may be corrupted."
+    exit 1
+fi
+"$SCRIPT_DIR"/weight.jq "$USDA_DIR"/$WEIGHT_FILE |
+    mongoimport --drop -d $DB -c $COLLECTION 

scripts/weight.jq

@@ -0,0 +1,23 @@
+#!/usr/bin/jq -Rf
+
+# How to use:
+# `scripts/weight.jq usda/WEIGHT.txt`
+# That's it.
+
+  split("\n")[]                  # split string into lines
+| split("^")                     # split lines int columns
+| {                              # format array of columns into jsons
+   "ndb_no": .[0] | ltrimstr("~") | rtrimstr("~"),
+   "seq": .[1] | tonumber,
+   "amount": .[2] | tonumber,
+   "msre_desc": .[3] | ltrimstr("~") | rtrimstr("~"),
+   "gm_wgt": .[4] | tonumber,
+   "num_data_pts": (if (.[5] | length) == 0 then null else .[5] | tonumber end),
+   "std_dev": (
+     if (.[6] | rtrimstr("\r") | length) == 0 then
+       null
+     else
+       .[6] | tonumber
+     end
+   )
+ }

src/name/tflucke/ieat2/controllers/WeightController.java

@@ -0,0 +1,45 @@
+package name.tflucke.ieat2.controllers;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import name.tflucke.ieat2.models.Weight;
+
+import org.springframework.beans.factory.annotation.Value;
+
+/**
+ * Provides APIs for reading USDA Weight objects.
+ *
+ * @author Thomas Flucke
+ * @since 2.0.0
+ */
+@RestController("/usda/weight")
+public class WeightController extends AbstractController<Weight> {
+
+    /**
+     * Creates a new controller
+     */
+    public WeightController() {
+        super(Weight.class);
+    }
+    
+    //@Override
+    @GetMapping("/usda/weight/{ndbno}")
+    public List<Weight> list(@PathVariable("ndbno") final String ndbno) {
+        return db.find(Weight.class).field("ndb_no").equal(ndbno).asList();
+    }
+
+    @Override
+    @GetMapping("/usda/weight")
+    public List<Weight> list() {
+        return super.list();
+    }
+}

src/name/tflucke/ieat2/models/Food.java

@@ -14,6 +14,7 @@ public abstract class Food extends DBObject {
     @JsonProperty("food_group")
     public String foodGroup;
     public boolean dry = unitType != Unit.Type.volume;
+    public float density = Float.NaN;
 
     public String getType() {
         return getClass().getSimpleName();

src/name/tflucke/ieat2/models/Weight.java

@@ -0,0 +1,14 @@
+package name.tflucke.ieat2.models;
+
+import org.mongodb.morphia.annotations.Entity;
+
+@Entity("usda-weight")
+public class Weight extends DBObject {
+    public String ndb_no;
+    public short seq;
+    public short amount;
+    public String msre_desc;
+    public int gm_wgt;
+    public Integer num_data_pt;
+    public Float std_dev;
+}