first commit

punctuation/heuristic.go

@@ -69,14 +69,6 @@ func heuristicEN(s string) string {
     return s
 }
 
-func hasTerminalPunct(s string) bool {
-    s = strings.TrimSpace(s)
-    if s == "" {
-        return false
-    }
-    r, _ := utf8.DecodeLastRuneInString(s)
-    return r == '.' || r == '?' || r == '!' || r == '…'
-}
 
 func ensureTerminalPunct(s string) string {
     if hasTerminalPunct(s) {

punctuation/normalize.go

@@ -0,0 +1,70 @@
+package punctuation
+
+import (
+    "strings"
+    "unicode"
+    "unicode/utf8"
+)
+
+// terminalPunctRunes — знаки, после которых не добавляем ещё одну фразовую точку.
+var terminalPunctRunes = map[rune]bool{
+    '.': true, '?': true, '!': true, '…': true,
+    ',': true, ';': true, ':': true,
+    ')': true, ']': true, '"': true, '\'': true,
+    '»': true, '”': true, '’': true,
+    '。': true, '，': true, '？': true, '！': true,
+}
+
+// CleanExcessive collapses duplicate and conflicting punctuation marks.
+func CleanExcessive(s string) string {
+    s = strings.TrimSpace(s)
+    if s == "" {
+        return s
+    }
+    var b strings.Builder
+    b.Grow(len(s))
+    prevClass := 0 // 0 none, 1 comma-like, 2 end, 3 other punct
+    for i := 0; i < len(s); {
+        r, size := utf8.DecodeRuneInString(s[i:])
+        cls := punctClass(r)
+        if cls != 0 && cls == prevClass {
+            i += size
+            continue
+        }
+        if cls == 2 && prevClass == 1 {
+            // drop sentence end right after comma-like (e.g. "привет,.")
+            i += size
+            continue
+        }
+        b.WriteRune(r)
+        if cls != 0 {
+            prevClass = cls
+        } else if !unicode.IsSpace(r) {
+            prevClass = 0
+        }
+        i += size
+    }
+    return strings.TrimSpace(b.String())
+}
+
+func punctClass(r rune) int {
+    switch r {
+    case ',', '，', '、', '،', ';', '؛', ':':
+        return 1
+    case '.', '?', '!', '…', '。', '？', '！':
+        return 2
+    }
+    if unicode.IsPunct(r) {
+        return 3
+    }
+    return 0
+}
+
+func hasTerminalPunct(s string) bool {
+    s = strings.TrimSpace(s)
+    if s == "" {
+        return false
+    }
+    r, _ := utf8.DecodeLastRuneInString(s)
+    return terminalPunctRunes[r]
+}

punctuation/normalize_test.go

@@ -0,0 +1,46 @@
+package punctuation
+
+import (
+    "context"
+    "strings"
+    "testing"
+)
+
+func TestCleanExcessive(t *testing.T) {
+    cases := []struct {
+        in, want string
+    }{
+        {"привет,,", "привет,"},
+        {"привет,.", "привет,"},
+        {"hello..", "hello."},
+        {"what??", "what?"},
+        {"ok!!!", "ok!"},
+        {"а. б. в.", "а. б. в."},
+    }
+    for _, tc := range cases {
+        got := CleanExcessive(tc.in)
+        if got != tc.want {
+            t.Errorf("CleanExcessive(%q) = %q, want %q", tc.in, got, tc.want)
+        }
+    }
+}
+
+func TestHasTerminalPunct_comma(t *testing.T) {
+    if !hasTerminalPunct("привет,") {
+        t.Fatal("comma should count as terminal for heuristic")
+    }
+    if hasTerminalPunct("привет") {
+        t.Fatal("bare word should not")
+    }
+}
+
+func TestHeuristic_noCommaPeriod(t *testing.T) {
+    h := Heuristic{}
+    out, err := h.Restore(context.Background(), "привет, мир", "ru")
+    if err != nil {
+        t.Fatal(err)
+    }
+    if strings.Contains(out, ",.") {
+        t.Fatalf("unexpected comma+period: %q", out)
+    }
+}

punctuation/punctuation.go

@@ -116,7 +116,11 @@ func Apply(ctx context.Context, r Restorer, enabled bool, text, language string)
     if text == "" {
         return text, nil
     }
-    return r.Restore(ctx, text, language)
+    out, err := r.Restore(ctx, text, language)
+    if err != nil {
+        return "", err
+    }
+    return CleanExcessive(out), nil
 }
 
 func Close(r Restorer) {