WikiLeaks_Analysis/build_token_frequency_table.R at master · drewconway/WikiLeaks_Analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
source('load_data.R')

get.tokens <- function(text, token.style = 'alphabetical')
{
  text <- tolower(text)

  regex <- ''

  if (token.style == 'alphabetical')
  {
    regex <- "[a-z]+"
  }

  if (token.style == 'extended-alphabetical')
  {
    regex <- "[a-z]+[a-z\\-']*[a-z]*"
  }

  tokens <- grep(regex, strsplit(text, '\\s+', perl = TRUE)[[1]], value = TRUE, perl = TRUE)

  return(tokens)
}

index.tokens <- function(text)
{
  tokens <- get.tokens(text)

  index <- list()

  for (token in tokens)
  {
    if (is.null(index[[token]]))
    {
      index[[token]] = 1
    }
    else
    {
      index[[token]] = index[[token]] + 1
    }
  }

  return(index)
}

raw.text <- paste(with(afg, Summary), collapse = ' ')
token.index <- index.tokens(raw.text)
lexical.database <- as.data.frame(token.index)
names(lexical.database) <- c('Token', 'Occurrences')
write.table(lexical.database,
            file = 'frequency_table.csv',
            sep = '\t',
            row.names = FALSE)