first commit

punctuation/internal/spwrap/sp.go

@@ -0,0 +1,92 @@
+//go:build xlm
+
+package spwrap
+
+/*
+#cgo CXXFLAGS: -std=c++17
+#cgo LDFLAGS: -lsentencepiece
+#include <stdlib.h>
+#include "sp_wrap.h"
+*/
+import "C"
+import (
+    "fmt"
+    "unsafe"
+)
+
+type Processor struct {
+    p *C.SPProcessor
+}
+
+func Load(path string) (*Processor, error) {
+    cpath := C.CString(path)
+    defer C.free(unsafe.Pointer(cpath))
+    var errMsg *C.char
+    p := C.sp_load(cpath, &errMsg)
+    if p == nil {
+        if errMsg != nil {
+            defer C.free(unsafe.Pointer(errMsg))
+            return nil, fmt.Errorf("sentencepiece: %s", C.GoString(errMsg))
+        }
+        return nil, fmt.Errorf("sentencepiece: failed to load %s", path)
+    }
+    return &Processor{p: p}, nil
+}
+
+func (proc *Processor) Close() {
+    if proc.p != nil {
+        C.sp_free(proc.p)
+        proc.p = nil
+    }
+}
+
+func (proc *Processor) BOSID() int {
+    return int(C.sp_bos_id(proc.p))
+}
+
+func (proc *Processor) EOSID() int {
+    return int(C.sp_eos_id(proc.p))
+}
+
+func (proc *Processor) PadID() int {
+    return int(C.sp_pad_id(proc.p))
+}
+
+func (proc *Processor) EncodeAsIDs(text string) ([]int, error) {
+    ctext := C.CString(text)
+    defer C.free(unsafe.Pointer(ctext))
+    var ids *C.int
+    var n C.int
+    var errMsg *C.char
+    if C.sp_encode(proc.p, ctext, &ids, &n, &errMsg) == 0 {
+        if errMsg != nil {
+            defer C.free(unsafe.Pointer(errMsg))
+            return nil, fmt.Errorf("sentencepiece encode: %s", C.GoString(errMsg))
+        }
+        return nil, fmt.Errorf("sentencepiece encode failed")
+    }
+    if ids == nil || n == 0 {
+        return nil, nil
+    }
+    defer C.free(unsafe.Pointer(ids))
+    out := make([]int, int(n))
+    slice := unsafe.Slice(ids, int(n))
+    for i := range out {
+        out[i] = int(slice[i])
+    }
+    return out, nil
+}
+
+func (proc *Processor) IDToPiece(id int) (string, error) {
+    var errMsg *C.char
+    piece := C.sp_id_to_piece(proc.p, C.int(id), &errMsg)
+    if piece == nil {
+        if errMsg != nil {
+            defer C.free(unsafe.Pointer(errMsg))
+            return "", fmt.Errorf("sentencepiece id to piece: %s", C.GoString(errMsg))
+        }
+        return "", fmt.Errorf("sentencepiece id to piece failed")
+    }
+    defer C.free(unsafe.Pointer(piece))
+    return C.GoString(piece), nil
+}

punctuation/internal/spwrap/sp_wrap.cc

@@ -0,0 +1,88 @@
+#include "sp_wrap.h"
+
+#include <sentencepiece_processor.h>
+
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+struct SPProcessor {
+    sentencepiece::SentencePieceProcessor proc;
+};
+
+static char *copy_err(const std::string &msg) {
+    char *out = static_cast<char *>(std::malloc(msg.size() + 1));
+    if (out != nullptr) {
+        std::memcpy(out, msg.c_str(), msg.size() + 1);
+    }
+    return out;
+}
+
+SPProcessor *sp_load(const char *path, char **err) {
+    if (err != nullptr) {
+        *err = nullptr;
+    }
+    auto *p = new SPProcessor();
+    const auto status = p->proc.Load(path);
+    if (!status.ok()) {
+        if (err != nullptr) {
+            *err = copy_err(status.ToString());
+        }
+        delete p;
+        return nullptr;
+    }
+    return p;
+}
+
+void sp_free(SPProcessor *p) { delete p; }
+
+int sp_bos_id(const SPProcessor *p) { return p->proc.bos_id(); }
+
+int sp_eos_id(const SPProcessor *p) { return p->proc.eos_id(); }
+
+int sp_pad_id(const SPProcessor *p) { return p->proc.pad_id(); }
+
+int sp_encode(const SPProcessor *p, const char *text, int **out_ids, int *out_len, char **err) {
+    if (err != nullptr) {
+        *err = nullptr;
+    }
+    if (out_ids != nullptr) {
+        *out_ids = nullptr;
+    }
+    if (out_len != nullptr) {
+        *out_len = 0;
+    }
+    std::vector<int> ids;
+    const auto status = p->proc.Encode(text, &ids);
+    if (!status.ok()) {
+        if (err != nullptr) {
+            *err = copy_err(status.ToString());
+        }
+        return 0;
+    }
+    if (ids.empty()) {
+        return 1;
+    }
+    int *buf = static_cast<int *>(std::malloc(sizeof(int) * ids.size()));
+    if (buf == nullptr) {
+        if (err != nullptr) {
+            *err = copy_err("malloc failed");
+        }
+        return 0;
+    }
+    for (size_t i = 0; i < ids.size(); i++) {
+        buf[i] = ids[i];
+    }
+    *out_ids = buf;
+    *out_len = static_cast<int>(ids.size());
+    return 1;
+}
+
+char *sp_id_to_piece(const SPProcessor *p, int id, char **err) {
+    if (err != nullptr) {
+        *err = nullptr;
+    }
+    const std::string piece = p->proc.IdToPiece(id);
+    return copy_err(piece);
+}

punctuation/internal/spwrap/sp_wrap.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SPProcessor SPProcessor;
+
+SPProcessor *sp_load(const char *path, char **err);
+void sp_free(SPProcessor *p);
+
+int sp_bos_id(const SPProcessor *p);
+int sp_eos_id(const SPProcessor *p);
+int sp_pad_id(const SPProcessor *p);
+
+int sp_encode(const SPProcessor *p, const char *text, int **out_ids, int *out_len, char **err);
+char *sp_id_to_piece(const SPProcessor *p, int id, char **err);
+
+#ifdef __cplusplus
+}
+#endif