tmain.go: parse cpt data - ags-upload - Insert AGS files to a database
(HTM) git clone git://src.adamsgaard.dk/ags-upload
(DIR) Log
(DIR) Files
(DIR) Refs
---
(DIR) commit d7aff58399bcb2d8e0573c9e027229bb9d0fd3f3
(DIR) parent 12b840e26e1e7437fea2415dc2794eb9a1c932cd
(HTM) Author: Anders Damsgaard <anders@adamsgaard.dk>
Date: Wed, 8 Oct 2025 11:40:06 +0200
main.go: parse cpt data
Diffstat:
M cmd/main.go | 171 ++++++++++++++++++++++---------
1 file changed, 123 insertions(+), 48 deletions(-)
---
(DIR) diff --git a/cmd/main.go b/cmd/main.go
t@@ -9,7 +9,8 @@ import (
"net/http"
"os"
"strings"
- "time"
+ "strconv"
+ //"time"
"github.com/gin-gonic/gin"
"gorm.io/driver/postgres"
t@@ -26,11 +27,24 @@ type CptInfo struct {
Contractor string // PROJ_CONT
}
-func ParseAGS(r io.Reader) (*CptInfo, error) {
+type Cpt struct { // group SCPG - data
+ ID uint `gorm:"primaryKey"`
+ InfoId uint //foreign key from CptInfo
+ LocationId string // LOCA_ID
+ TestReference string // SCPG_TESN
+ Depth float64 // SCPT_DPTH
+ ConeRes float64 // SCPT_RES
+ SideFric float64 // SCPT_FRES
+ Pore1 float64 // SCPT_PWP1
+ Pore2 float64 // SCPT_PWP2
+ Pore3 float64 // SCPT_PWP3
+ FrictionRatio float64 // SCPT_FRR
+}
+func ParseAGSProjectAndSCPT(r io.Reader) (*CptInfo, []Cpt, error) {
norm, err := dos2unix(r)
if err != nil {
- return nil, fmt.Errorf("read: %w", err)
+ return nil, nil, fmt.Errorf("read: %w", err)
}
cr := csv.NewReader(norm)
t@@ -38,69 +52,113 @@ func ParseAGS(r io.Reader) (*CptInfo, error) {
cr.LazyQuotes = true
var (
- inPROJ bool
- headerIndex map[string]int
+ curGroup string
+ headersByGrp = map[string]map[string]int{} // GROUP -> header index map
+ project *CptInfo
+ cpts []Cpt
)
+ get := func(group string, data []string, name string) string {
+ hm := headersByGrp[group]
+ if hm == nil {
+ return ""
+ }
+ if idx, ok := hm[strings.ToUpper(name)]; ok && idx >= 0 && idx < len(data) {
+ return data[idx]
+ }
+ return ""
+ }
+ parseF64 := func(s string) float64 {
+ if s == "" {
+ return 0
+ }
+ // Optional: handle decimal commas
+ s = strings.ReplaceAll(s, ",", ".")
+ f, _ := strconv.ParseFloat(s, 64)
+ return f
+ }
+
for {
rec, err := cr.Read()
if err == io.EOF {
break
}
if err != nil {
- return nil, fmt.Errorf("csv: %w", err)
+ return nil, nil, fmt.Errorf("csv: %w", err)
}
if len(rec) == 0 {
continue
}
-
for i := range rec {
rec[i] = strings.TrimSpace(rec[i])
+ // Some exporters put empty quotes => "" — leave as empty string
}
- switch strings.ToUpper(rec[0]) {
+ tag := strings.ToUpper(rec[0])
+ switch tag {
case "GROUP":
- inPROJ = len(rec) > 1 && strings.EqualFold(rec[1], "PROJ")
- headerIndex = nil
+ if len(rec) > 1 {
+ curGroup = strings.ToUpper(strings.TrimSpace(rec[1]))
+ } else {
+ curGroup = ""
+ }
case "HEADING":
- if !inPROJ {
+ if curGroup == "" {
continue
}
- headerIndex = make(map[string]int)
+ m := make(map[string]int, len(rec)-1)
for i := 1; i < len(rec); i++ {
key := strings.ToUpper(strings.TrimSpace(rec[i]))
- headerIndex[key] = i - 1 // positions in the "DATA" slice after skipping the first token
+ m[key] = i - 1 // position in DATA after skipping tag
}
+ headersByGrp[curGroup] = m
+
case "DATA":
- if !inPROJ || headerIndex == nil {
+ if curGroup == "" {
continue
}
- data := rec[1:] // align with headerIndex positions
+ data := rec[1:]
- get := func(h string) string {
- if idx, ok := headerIndex[strings.ToUpper(h)]; ok && idx >= 0 && idx < len(data) {
- return data[idx]
+ switch curGroup {
+ case "PROJ":
+ if project != nil {
+ // If multiple PROJ rows exist, keep the first (typical).
+ continue
+ }
+ project = &CptInfo{
+ SourceId: get("PROJ", data, "PROJ_ID"),
+ Name: get("PROJ", data, "PROJ_NAME"),
+ Location: get("PROJ", data, "PROJ_LOC"),
+ Client: get("PROJ", data, "PROJ_CLNT"),
+ Contractor: get("PROJ", data, "PROJ_CONT"),
}
- return ""
- }
- p := &CptInfo{
- SourceId: get("PROJ_ID"),
- Name: get("PROJ_NAME"),
- Location: get("PROJ_LOC"),
- Client: get("PROJ_CLNT"),
- Contractor: get("PROJ_CONT"),
+ case "SCPT":
+ cpts = append(cpts, Cpt{
+ LocationId: get("SCPT", data, "LOCA_ID"),
+ TestReference: get("SCPT", data, "SCPG_TESN"),
+ Depth: parseF64(get("SCPT", data, "SCPT_DPTH")),
+ ConeRes: parseF64(get("SCPT", data, "SCPT_RES")),
+ SideFric: parseF64(get("SCPT", data, "SCPT_FRES")),
+ Pore1: parseF64(get("SCPT", data, "SCPT_PWP1")),
+ Pore2: parseF64(get("SCPT", data, "SCPT_PWP2")),
+ Pore3: parseF64(get("SCPT", data, "SCPT_PWP3")),
+ FrictionRatio: parseF64(get("SCPT", data, "SCPT_FRR")),
+ })
+ default:
+ // ignore other groups for now
}
- return p, nil
+ // ignore UNIT, TYPE, etc.
default:
continue
}
}
- return nil, fmt.Errorf("no data found")
+ return project, cpts, nil
}
+
func dos2unix(r io.Reader) (io.Reader, error) {
all, err := io.ReadAll(r)
if err != nil {
t@@ -135,44 +193,61 @@ func main() {
log.Fatal(err)
}
+ if err := db.AutoMigrate(&Cpt{}); err != nil {
+ log.Fatal(err)
+ }
+
r := gin.Default()
// ~32 MB file cap for multipart
r.MaxMultipartMemory = 32 << 20
r.POST("/ingest/ags", func(c *gin.Context) {
- reader, cleanup, err := getAGSReader(c.Request)
+ file, _, err := c.Request.FormFile("file")
if err != nil {
- c.String(http.StatusBadRequest, "upload error: %v", err)
+ c.String(400, "missing multipart file: %v", err)
return
}
- if cleanup != nil {
- defer cleanup()
- }
+ defer file.Close()
- p, err := ParseAGS(reader)
+ proj, cpts, err := ParseAGSProjectAndSCPT(file)
if err != nil {
- c.String(http.StatusBadRequest, "parse error: %v", err)
+ c.String(400, "parse error: %v", err)
return
}
- err = db.
- Where("source_id = ?", p.SourceId).
- Assign(p).
- FirstOrCreate(p).Error
+ err = db.Transaction(func(tx *gorm.DB) error {
+
+ // Upsert project by SourceId (make SourceId unique if you rely on it)
+ if proj != nil {
+ if err := tx.
+ Where("source_id = ?", proj.SourceId).
+ Assign(proj).
+ FirstOrCreate(proj).Error; err != nil {
+ return err
+ }
+ }
+
+ // If you later derive InfoId from a SC* info table, set it here before insert.
+ if len(cpts) > 0 {
+ // Optional: add a foreign key to project if you want (e.g., ProjectID)
+ // for i := range cpts { cpts[i].ProjectID = proj.ID }
+
+ if err := tx.CreateInBatches(cpts, 2000).Error; err != nil {
+ return err
+ }
+ }
+
+ return nil
+ })
if err != nil {
- c.String(http.StatusInternalServerError, "db error: %v", err)
+ c.String(500, "db error: %v", err)
return
}
- c.JSON(http.StatusCreated, gin.H{
- "id": p.ID,
- "sourceId": p.SourceId,
- "name": p.Name,
- "location": p.Location,
- "client": p.Client,
- "contractor": p.Contractor,
- "savedAt": time.Now().Format(time.RFC3339),
+ c.JSON(201, gin.H{
+ "project": proj,
+ "cpts": len(cpts),
})
})