计算文本的语言具体性 | 以JCR2021论文为例

We computed a concreteness score for each conversational turn (averaging across all words in that turn) and for each conversational participant (averaging across all words over all their turns). Results were the same whether or not stop words commonly excluded from linguistics analyses (e.g., but, and) were included. We report results excluding stop words.

import pandas as pd
from nltk.tokenize import word_tokenize

#JCR文中使用的Paetzold2016的词典
# Paetzold2016文中的词典下载链接失效。这里使用Brysbaert2014的词典
df = pd.read_excel("Concreteness_ratings_Brysbaert_et_al_BRM.xlsx")

from nltk.tokenize import word_tokenize

def query_concreteness(word):
    """
    查询word的具体性得分
    """
    try:
        return df[df["Word"]==word]['Conc.M'].values[0]
    except:
        return 0
    
 

def concreteness_score(text):
    """
    计算文本的具体性得分
    """
    score = 0
    text = text.lower()
    
    try:
        words = word_tokenize(text)
    except:
        print('你的电脑nltk没配置好，请观看视频https://www.bilibili.com/video/BV14A411i7DB')
        words = text.split(' ')
        
    for word in words:
        try:
            score += query_concreteness(word=word)
        except:
            score += 0
            
    return score/len(words)
    
  
# 案例
employee_replys = ["I'll go look for that",
                   "I'll go search for that",
                   "I'll go search for that top",
                   "I'll go search for that t-shirt",
                   "I'll go look for that t-shirt in grey",
                   "I'll go search for that t-shirt in grey"]

for idx, reply in enumerate(employee_replys):
    score=concreteness_score(reply)
    template = "Concreteness Score: {score:.2f} | Example-{idx}: {exmaple}"
    print(template.format(score=score, 
                          idx=idx, 
                          exmaple=reply))

Concreteness Score: 1.55 | Example-0: I'll go look for that
Concreteness Score: 1.55 | Example-1: I'll go search for that
Concreteness Score: 1.89 | Example-2: I'll go search for that top
Concreteness Score: 2.04 | Example-3: I'll go search for that t-shirt
Concreteness Score: 2.37 | Example-4: I'll go look for that t-shirt in grey
Concreteness Score: 2.37 | Example-5: I'll go search for that t-shirt in grey

pip3 install cntext==1.7.7

import cntext as ct

# load the concreteness.pkl dictionary file
concreteness_df = ct.load_pkl_dict('Concreteness.pkl')['Concreteness']
concreteness_df.head()

reply = "I'll go look for that"

score=ct.sentiment_by_valence(text=reply, 
                              diction=concreteness_df, 
                              lang='english')
score

1.85

employee_replys = ["I'll go look for that",
                   "I'll go search for that",
                   "I'll go search for that top",
                   "I'll go search for that t-shirt",
                   "I'll go look for that t-shirt in grey",
                   "I'll go search for that t-shirt in grey"]

for idx, reply in enumerate(employee_replys):
    score=ct.sentiment_by_valence(text=reply, 
                                  diction=concreteness_df, 
                                  lang='english')
    
    template = "Concreteness Score: {score:.2f} | Example-{idx}: {exmaple}"
    print(template.format(score=score, 
                          idx=idx, 
                          exmaple=reply))
    
ct.sentiment_by_valence(text=text, diction=concreteness_df, lang='english')

Concreteness Score: 1.55 | Example-0: I'll go look for that
Concreteness Score: 1.55 | Example-1: I'll go search for that
Concreteness Score: 1.89 | Example-2: I'll go search for that top
Concreteness Score: 2.04 | Example-3: I'll go search for that t-shirt
Concreteness Score: 2.37 | Example-4: I'll go look for that t-shirt in grey
Concreteness Score: 2.37 | Example-5: I'll go search for that t-shirt in grey