tests/testthat/_snaps/collapse_tokens.md

collapse_tokens works

{
  "type": "list",
  "attributes": {
    "names": {
      "type": "character",
      "attributes": {},
      "value": ["doc_id", "sentence_id", "token_id", "token"]
    },
    "row.names": {
      "type": "integer",
      "attributes": {},
      "value": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
    },
    "class": {
      "type": "character",
      "attributes": {},
      "value": ["tbl_df", "tbl", "data.frame"]
    }
  },
  "value": [
    {
      "type": "integer",
      "attributes": {
        "levels": {
          "type": "character",
          "attributes": {},
          "value": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50"]
        },
        "class": {
          "type": "character",
          "attributes": {},
          "value": ["factor"]
        }
      },
      "value": [1, 1, 1, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]
    },
    {
      "type": "integer",
      "attributes": {},
      "value": [1, 1, 1, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]
    },
    {
      "type": "integer",
      "attributes": {},
      "value": [1, 2, 3, 1, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
    },
    {
      "type": "character",
      "attributes": {},
      "value": ["銀河鉄道", "の", "夜", "宮沢賢治", "一", " ", "午後", "の", "授業", "「", "では", "みなさん", "は", "、", "そういう", "ふう", "に", "川", "だ", "と", "言わ", "れ", "たり", "、", "乳", "の", "流れ", "た", "あと", "だ", "と", "言わ", "れ", "たり"]
    }
  ]
}


paithiov909/gibasa documentation built on June 14, 2025, 4:31 p.m.