Added ability to parse recipes from external websites.

Currently requires the source web site's format to be hard-coded into a
parser object.  Still TODO is move these objects into a database and
allow an admin to define new ones interactively.

Added ability to retractively edit the food ID in an ingredient.

server/app/com/weEat/controllers/ParserController.scala

@@ -0,0 +1,120 @@
+package com.weEat.controllers
+
+import com.weEat.shared.models._
+import javax.inject.{Inject,Singleton}
+import play.api.libs.json._
+import play.api.mvc._
+import scala.concurrent.Future
+import com.weEat.models.Authorization
+import scalaoauth2.provider.{AuthInfoRequest,OAuth2ProviderActionBuilders}
+import com.weEat.services.OAuth2Service
+import net.ruippeixotog.scalascraper.browser.JsoupBrowser
+import net.ruippeixotog.scalascraper.dsl.DSL._
+import net.ruippeixotog.scalascraper.dsl.DSL.Extract._
+import net.ruippeixotog.scalascraper.model.Element
+import net.ruippeixotog.scalascraper.scraper.HtmlExtractor
+
+@Singleton
+class ParserController @Inject()(
+  val controllerComponents: ControllerComponents,
+  oauth: OAuth2Service,
+  usdaController: USDAController,
+  foodController: FoodController
+) extends BaseController
+    with OAuth2ProviderActionBuilders {
+  implicit val ec = scala.concurrent.ExecutionContext.global
+
+  private val _browser = JsoupBrowser()
+
+  def parseURL() = AuthorizedAction[Authorization](oauth).async(parse.text)({ implicit request: AuthInfoRequest[String, Authorization] =>
+    val url = request.body
+    _findParser(url).fold(Future.successful(NotFound(s"No parser available for $url."))) { (parser) =>
+      val doc = _browser.get(url)
+      val title = doc >> parser.titleExtractor
+      val servings = doc >> parser.servingExtractor
+      val prepTime = parser.prepTimeExtractor.map(doc >> _)
+      val cookTime = parser.cookTimeExtractor.map(doc >> _)
+      val ingredients = doc >> parser.ingredientExtractor
+      val instructions = doc >> parser.instructionExtractor
+
+      Future.sequence(ingredients.map(_parseIngredient _))
+        .map((ingredients) => Ok(Json.toJson(RecipeNodeNoId(
+          title,
+          servings.getOrElse(1.0f),
+          1.0f,
+          UnitType.NUMBER,
+          ingredients.toSeq,
+          /* tflucke@[2023-10-26]: Do not pss along the instructions since this
+           * could be a violation of the Recipe Author's copyright. */
+          Nil, //instructions.toSeq,
+          None,
+          None,
+          Some(url)
+        ))))
+    }
+  })
+
+  private def _findParser(url: String): Option[Parser] = {
+    val host = new java.net.URL(url).getAuthority()
+    val hostNoWWW = if (host.startsWith("www.")) host.substring("www.".length) else host
+    Map(
+      ("epicurious.com" -> Parser.epicurious),
+      ("mccormick.com" -> Parser.mccormick)
+    ).get(hostNoWWW)
+  }
+
+  private def _parseIngredient(ingredientLine: String): Future[Ingredient] = {
+    val numberPattern = raw"(\d+)[\d-_]*\s(\w+)\s+(.+)".r
+    val fractionPattern = raw"(\d+)/(\d+)[\d-_]*\s(\w+)\s+(.+)".r
+
+    ingredientLine match {
+      case numberPattern(amount, unit, rest) =>
+        _guessFoodFromStr(rest).map(Ingredient(_, amount.toFloat, MeasureUnit.guessUnit(unit).getOrElse(Count)))
+      case fractionPattern(numerator, denominator, unit, rest) =>
+        _guessFoodFromStr(rest).map(Ingredient(_, numerator.toFloat/denominator.toFloat, MeasureUnit.guessUnit(unit).getOrElse(Count)))
+      case noUnitLine => _guessFoodFromStr(noUnitLine).map(Ingredient(_, 1, Count))
+    }
+
+  }
+
+  private def _guessFoodFromStr(foodLine: String): Future[Ingredient.IngredientId] = {
+    import gov.usda.nal.fdc.models.DataType._
+    usdaController.fdc.getFoodsSearch(foodLine, Seq(
+      Foundation, Survey, SRLegacy
+    ), pageSize = Some(10))().flatMap({ (fdcResult) =>
+      Future.sequence(fdcResult.foods.map((food) => foodController.getByFdcId(food.fdcId)))
+        .map(_.flatten
+          .headOption
+          .fold[Ingredient.IngredientId](Ingredient.USDAId(fdcResult.foods.head.fdcId))((foodNode) => Ingredient.FoodNodeId(foodNode._id))
+        )
+    })
+  }
+}
+
+case class Parser(
+  titleExtractor: HtmlExtractor[Element, String],
+  servingExtractor: HtmlExtractor[Element, Option[Float]],
+  prepTimeExtractor: Option[HtmlExtractor[Element, String]],
+  cookTimeExtractor: Option[HtmlExtractor[Element, String]],
+  ingredientExtractor: HtmlExtractor[Element, Iterable[String]],
+  instructionExtractor: HtmlExtractor[Element, Iterable[String]],
+)
+
+object Parser {
+  val mccormick = Parser(
+    text("h1"),
+    text(".main-title .count").map(_.toFloatOption),
+    Some(text(".prep_time .first_content")),
+    cookTimeExtractor = Some(text(".ingredients .first_content")),
+    ingredientExtractor = texts(".recipe-about-list li"),
+    texts(".instructions-main span.para")
+  )
+  val epicurious = Parser(
+    text("h1"),
+    text("""div[data-testid="IngredientList"] > p""").map("Yield: \\D*(\\d+).*".r.findFirstMatchIn(_).map(_.group(1).toFloat)),
+    None,
+    None,
+    texts("""div[data-testid="IngredientList"] > div > div"""),
+    texts("""div[data-testid="InstructionsWrapper"] > ol > li > p""")
+  )
+}

server/conf/routes

@@ -57,18 +57,24 @@ GET  /v1/food/     com.weEat.controllers.FoodController.all()
 # type: com.weEat.shared.models.FoodNodeId
 GET  /v1/food/:id  com.weEat.controllers.FoodController.get(id: String)
 
-# Share Route
+# Shared Route
 # type: com.weEat.shared.models.FoodNodeId
 #GET  /v1/food/:id/image/:img  com.weEat.controllers.FoodController.get(id: String, img: String)
 
-# Share Route
+# Shared Route
 # type: com.weEat.shared.models.FoodNodeId
 #PUT  /v1/food/:id/image  com.weEat.controllers.FoodController.addImageTo(id: String)
 
-# Share Route
+# Shared Route
 #DELETE  /v1/food/:id/image/:img  com.weEat.controllers.FoodController.deleteImage(id: String, img: String)
 
-# Share Route
+# Shared Route
+# content: text
+# body: String
+# type: com.weEat.shared.models.RecipeNodeNoId
+POST  /v1/food/recipe/parse com.weEat.controllers.ParserController.parseURL()
+
+# Shared Route
 # type: gov.usda.nal.fdc.models.FoodItem
 #GET   /fdc/food   com.weEat.controllers.USDAController.getFoods(id: String, fmt: String ?= "Full")
 

shared/shared/src/main/scala/com/weEat/shared/models/FoodNode.scala

@@ -68,6 +68,13 @@ object Ingredient {
   case class USDAId(id: Long) extends IngredientId {
     def typ = "usda"
   }
+  object IngredientId {
+    def fromFoodNode(node: FoodNode) = node match {
+      case food: FoodNodeWithId => FoodNodeId(food._id)
+      case food: USDANodeNoId => USDAId(food.fdcId)
+      case _ => ???
+    }
+  }
 
   implicit def foodNodeToId(food: FoodNodeWithId) = FoodNodeId(food._id)
   implicit def foodItemToId(food: FoodItem) = USDAId(food.fdcId)
@@ -85,11 +92,7 @@ object Ingredient {
     unit: MeasureUnit
   ): Ingredient = {
     class CachedIngredient(node: FoodNode, amount: Float, unit: MeasureUnit)
-        extends Ingredient(node match {
-          case food: FoodNodeWithId => FoodNodeId(food._id)
-          case food: USDANodeNoId => USDAId(food.fdcId)
-          case _ => ???
-        }, amount, unit) {
+        extends Ingredient(IngredientId.fromFoodNode(node), amount, unit) {
 
       override lazy val food = Future.successful(node)
 
@@ -136,7 +139,7 @@ sealed trait RecipeNode extends FoodNode {
   )
 
   def nutrient(num: String) = ingredients.map({ ingr =>
-    ingr.food.map( f => 
+    ingr.food.map( (f) => 
       f.nutrient(num)
         .map(_ * (ingr.unit
           .convert(f, ingr.amount, f.defaultUnitType.defaultUnit) match {

webClient/src/main/scala/com/weEat/models/RecipeVar.scala

@@ -29,6 +29,7 @@ case class RecipeVar(recipe: Option[RecipeNode])
     Seq(),
     Seq(),
     None,
+    None,
     None
   ))
 
@@ -74,6 +75,7 @@ case class RecipeVar(recipe: Option[RecipeNode])
     case (None, "") => "100"
     case (Some(""), "") => "100"
     case (Some(_), str) => str
+    case (None, str) => str
   }).map(_.toFloatOption)
 
   private val _volumeUnit = Var[MeasureUnit](Milliliter)
@@ -115,6 +117,12 @@ case class RecipeVar(recipe: Option[RecipeNode])
       case (None, _) => None
     }
 
+  private val _sourceIn = input(idAttr := "source",
+    cls := "form-control",
+    defaultValue := _defaultRecipe.source.getOrElse("")
+  )
+  val source: Signal[Option[String]] = _sourceIn.value.map((s) => Option.when(s.size > 0)(s))
+
 
   private val internalFoodDS = Dataset(
     { _ => Nil},
@@ -141,59 +149,71 @@ case class RecipeVar(recipe: Option[RecipeNode])
 
   private val _ingredients = Var[Seq[Ingredient]](_defaultRecipe.ingredients)
   val ingredients = _ingredients.signal
-  private val _ingredientSearch = input(typ := "text",
+  private val _ingredientSearch = _ingredientInput() { (e) =>
+    e.selectable.map(_.data).foreach({ (node) =>
+      // TODO: default unit
+      _editIngredient(Ingredient.fromFoodNode(node, 0, Gram)) { (in) =>
+        _ingredients.update(_ :+ in)
+        e.target.asInstanceOf[HTMLInputElement].value = ""
+        Future.successful(())
+      }
+    })
+  }
+
+  private def _ingredientInput(defaul: Signal[Option[FoodNode]] = Val(None))(onSelect: (CursorEvent[FoodNode]) => Unit) = input(typ := "text",
     cls := "form-control input-sm",
+    value <-- defaul.optionMap(_.name).withDefault(""),
     onMountCallback({(ctx) =>
       val elm = ctx.thisNode
       TypeaheadElement[FoodNode](
         elm.ref.asInstanceOf[HTMLInputElement],
         minLength = 3
       )(Seq(internalFoodDS, usdaFoodDS))
-      elm.amend(
-        Typeahead.onSelected[FoodNode] --> { (e: CursorEvent[FoodNode]) =>
-          e.selectable.map(_.data).foreach({ node =>
-            // TODO: default unit
-            _editIngredient(Ingredient.fromFoodNode(node, 0, Gram)) { in =>
-              _ingredients.update(_ :+ in)
-              elm.ref.value = ""
-              Future.successful(())
-            }
-          })
-        }
-      )
+      elm.amend(Typeahead.onSelected[FoodNode] --> onSelect)
     })
   )
 
   private def _editIngredient(ing: Ingredient)(
     callback: (Ingredient => Future[_])
   ) = {
+    val id = Var[Ingredient.IngredientId](ing.id)
     val amount = Var[Float](ing.amount)
-    val unit = Var[MeasureUnit](Gram)
+    val unit = Var[MeasureUnit](ing.unit)
 
     Overlay.confirmFuture(ing.food.map({ (food) =>
       val amountIn = input(typ := "number",
-        cls := "form-control input-sm col-9",
+        cls := "form-control input-sm",
         minAttr := "0",
         defaultValue := ing.amount.toString,
         onInput.mapToValue.map(_.toFloat) --> amount
       )
-      val unitIn = select(cls := "col-3 custom-select",
+      val unitIn = select(cls := "custom-select input-group-append",
         MeasureUnit.units.zipWithIndex.map({ case (unit, idx) =>
           // TODO: default selected dynamic
           option(value := idx.toString,
-            selected := unit.abr == Gram.abr,
+            selected := unit == ing.unit,
             unit.name
           )
         }),
         onChange.mapToValue.map(_.toInt).map(MeasureUnit.units(_)) --> unit
       )
 
-      div(cls := "row", amountIn, unitIn)
+      div(cls := "row",
+        div(cls := "col-12 input-group",
+          _ingredientInput(Signal.fromFuture(ing.food)) { (e) =>
+            e.selectable.map(_.data).foreach({ (node) => id.set(Ingredient.IngredientId.fromFoodNode(node)) })
+          },
+          amountIn,
+          unitIn
+        )
+      )
     }),
-      amount.signal.recoverToTry.map(_.isSuccess) &&
+      id.signal.recoverToTry.map(_.isSuccess) &&
+        amount.signal.recoverToTry.map(_.isSuccess) &&
         unit.signal.recoverToTry.map(_.isSuccess)
     ) { () =>
       callback(ing.copy(
+        id = id.now(),
         amount = amount.now(),
         unit = unit.now()
       ))
@@ -300,13 +320,13 @@ case class RecipeVar(recipe: Option[RecipeNode])
 
   val newInstance: Signal[Try[RecipeNodeNoId]] =
     name.combineWithFn(stdQties, stdQtiesPServing, defaultUnitType,
-      ingredients, steps, density, massPerUnit.signal) {
+      ingredients, steps, density, massPerUnit.signal, source) {
       case (
         name, stdQties, stdQtiesPServing, defaultUnitType, ingredients, steps,
-        density, massPerUnit
+        density, massPerUnit, source
       ) => Success(RecipeNodeNoId(
         name, stdQties, stdQtiesPServing, defaultUnitType, ingredients, steps,
-        density, massPerUnit
+        density, massPerUnit, source
       ))
     }
 
@@ -314,8 +334,10 @@ case class RecipeVar(recipe: Option[RecipeNode])
     form(cls := "form-group",
       div(cls := "container",
         div(cls := "row",
-          div(cls := "col-md-12",
-            label("Name: "),
+          div(cls := "col-md-12 input-group",
+            div(cls := "input-group-prepend",
+              label(cls := "input-group-text", "Name")
+            ),
             _nameIn
           )
         ),

webClient/src/main/scala/com/weEat/modules/Overlay.scala

@@ -56,7 +56,6 @@ object Overlay {
     top             := "50%",
     transform       := "translateY(-50%)",
     maxHeight       := "80%",
-    overflowY       := "auto",
     display         := "inline-block",
     verticalAlign   := "middle",
     maxWidth        := "50%"

webClient/src/main/scala/com/weEat/views/RecipeEdit.scala

@@ -62,32 +62,34 @@ object RecipeEdit extends View[Option[String]] {
           )
         ),
         child <-- refRecipe.map({
-          case Some(RecipeNodeId(id, uid, _, _, _, _, _, _, _, _)) =>
+          case Some(RecipeNodeId(id, uid, _, _, _, _, _, _, _, _, _)) =>
             button(cls := "btn",
-              onClick --> { (e: Event) =>
-                implicit val owner = new ManualOwner()
-
-                FoodController.update(id, Some(uid))(recipe.observe.now().get)
-                  .onComplete {
-                    case Success(recipe) =>
-                      View.router.pushState(RecipeView.ViewPage(recipe._id))
-                    case Failure(ex) =>
-                      println("Could not update recipe")
-                      throw ex
-                  }
+              onMountBind { (ctx) =>
+                onClick --> { (e: Event) =>
+                  implicit val owner = ctx.owner
+                  FoodController.update(id, Some(uid))(recipe.observe.now().get)
+                    .onComplete {
+                      case Success(recipe) =>
+                        View.router.pushState(RecipeView.ViewPage(recipe._id))
+                      case Failure(ex) =>
+                        println("Could not update recipe")
+                        throw ex
+                    }
+                }
               },
               "Update"
             )
           case _ => button(cls := "btn",
-            onClick --> { (e: Event) =>
-              implicit val owner = new ManualOwner()
-
-              FoodController.add()(recipe.observe.now().get).onComplete {
-                case Success(recipe) =>
-                  View.router.pushState(RecipeView.ViewPage(recipe._id))
-                case Failure(ex) =>
-                  println("Could not add recipe")
-                  throw ex
+            onMountBind { (ctx) =>
+              onClick --> { (e: Event) =>
+                implicit val owner = ctx.owner
+                FoodController.add()(recipe.observe.now().get).onComplete {
+                  case Success(recipe) =>
+                    View.router.pushState(RecipeView.ViewPage(recipe._id))
+                  case Failure(ex) =>
+                    println("Could not add recipe")
+                    throw ex
+                }
               }
             },
             "Add"

webClient/src/main/scala/com/weEat/views/RecipeImporter.scala

@@ -0,0 +1,81 @@
+package com.weEat.view
+
+import com.raquo.laminar.api.L._
+import com.raquo.waypoint._
+import com.weEat.controllers.{ParserController,FoodController}
+import com.weEat.models.RecipeVar
+import play.api.libs.json.{JsValue,JsNull}
+import org.scalajs.dom.URL
+import com.weEat.shared.models.RecipeNodeNoId
+import org.scalajs.dom.Event
+import scala.util.{Failure,Success}
+
+object RecipeImporter extends View[Unit] {
+  implicit val ec = com.weEat.shared.ctx
+
+  val navName = Some("Recipe Import")
+  val tag = "recipe-import"
+
+  case class ViewPage() extends P {
+    val title = "Recipe Import"
+    def jsonValue = JsNull
+  }
+  def parseJson(jsVal: JsValue) = ViewPage()
+  def route = Route.onlyQuery(
+    encode = (page: ViewPage) => (),
+    decode = (_: Unit) => ViewPage(),
+    pattern = (root / tag / endOfSegments) ? ignore
+  )
+  def defaultPage = ViewPage()
+
+  def content(p: Signal[ViewPage]) = {
+    val rawInput = Var("")
+    val urlParsedInput = rawInput.signal.map(new URL(_))
+    val parsedRecipe = Var[Option[RecipeNodeNoId]](None)
+    val recipeVar = parsedRecipe.signal.map(RecipeVar(_))
+
+    div(
+      h2("Recipe Import"),
+      div(cls := "input-group",
+        div(cls := "input-group-prepend",
+          label(cls := "input-group-text", forId := "srcUrl", "Source")
+        ),
+        input(typ := "text",
+          idAttr := "srcUrl",
+          cls := "form-control input-sm",
+          onInput.mapToValue --> rawInput
+        ),
+        div(cls := "input-group-append",
+          button(cls := "btn btn-light",
+            onMountBind { (c) =>
+              onClick --> { (_) =>
+                implicit val owner = c.owner
+                import com.weEat.Main.headers
+                Signal.fromFuture(ParserController.parseURL()(urlParsedInput.observe.now().toString)).addObserver(parsedRecipe.toObserver)
+              }
+            },
+            "Parse"
+          )
+        )
+      ),
+      child <-- recipeVar.map(_.render),
+      button(cls := "btn",
+        onMountBind { (ctx) =>
+          onClick --> { (e: Event) =>
+            import com.weEat.Main.headers
+            implicit val owner = ctx.owner
+            
+            FoodController.add()(recipeVar.flatMap(_.newInstance).observe.now().get).onComplete {
+              case Success(recipe) =>
+                View.router.pushState(RecipeView.ViewPage(recipe._id))
+              case Failure(ex) =>
+                println("Could not add recipe")
+                throw ex
+            }
+          }
+        },
+        "Add"
+      )
+    )
+  }
+}

webClient/src/main/scala/com/weEat/views/View.scala

@@ -41,6 +41,7 @@ object View {
     UsdaImporter,
     RecipeEdit,
     RecipeView,
+    RecipeImporter
     // UserManage,
     //ProfileView,
     //ProfileEdit
@@ -75,6 +76,7 @@ object View {
     .collectSignal[FoodSearch.ViewPage] { (page) => FoodSearch.content(page) }
     .collectSignal[RecipeEdit.ViewPage] { (page) => RecipeEdit.content(page) }
     .collectSignal[RecipeView.ViewPage] { (page) => RecipeView.content(page) }
+    .collectSignal[RecipeImporter.ViewPage] { (page) => RecipeImporter.content(page) }
 
 }