{"id":"https:\/\/openalex.org\/W4384390283","doi":"https:\/\/doi.org\/10.48550\/arxiv.2307.06869","title":"DecompEval: Evaluating Generated Texts as Unsupervised Decomposed Question Answering","display_name":"DecompEval: Evaluating Generated Texts as Unsupervised Decomposed Question Answering","publication_year":2023,"publication_date":"2023-07-13","ids":{"openalex":"https:\/\/openalex.org\/W4384390283","doi":"https:\/\/doi.org\/10.48550\/arxiv.2307.06869"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2307.06869","is_oa":true,"landing_page_url":"http:\/\/arxiv.org\/abs\/2307.06869","pdf_url":"https:\/\/arxiv.org\/pdf\/2307.06869","source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https:\/\/arxiv.org\/pdf\/2307.06869","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https:\/\/openalex.org\/A5091491018","display_name":"Pei Ke","orcid":"https:\/\/orcid.org\/0000-0001-7554-5636"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ke, Pei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5101488344","display_name":"Fei Huang","orcid":"https:\/\/orcid.org\/0000-0002-3709-5053"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Fei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5012014905","display_name":"Fei Mi","orcid":"https:\/\/orcid.org\/0000-0001-6358-9922"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mi, Fei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5115592503","display_name":"Yasheng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yasheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5103721236","display_name":"Qun Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Qun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5100452974","display_name":"Xiaoyan Zhu","orcid":"https:\/\/orcid.org\/0000-0002-7173-7793"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Xiaoyan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https:\/\/openalex.org\/A5044042138","display_name":"Minlie Huang","orcid":"https:\/\/orcid.org\/0000-0001-7111-1849"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Minlie","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https:\/\/openalex.org\/A5091491018"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https:\/\/openalex.org\/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1702","display_name":"Artificial Intelligence"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}},"topics":[{"id":"https:\/\/openalex.org\/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1702","display_name":"Artificial Intelligence"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}},{"id":"https:\/\/openalex.org\/T10181","display_name":"Natural Language Processing Techniques","score":0.9991000294685364,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1702","display_name":"Artificial Intelligence"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}},{"id":"https:\/\/openalex.org\/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9793999791145325,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https:\/\/openalex.org\/keywords\/interpretability","display_name":"Interpretability","score":0.9209123849868774},{"id":"https:\/\/openalex.org\/keywords\/automatic-summarization","display_name":"Automatic summarization","score":0.8266123533248901},{"id":"https:\/\/openalex.org\/keywords\/generalization","display_name":"Generalization","score":0.8227543234825134},{"id":"https:\/\/openalex.org\/keywords\/computer-science","display_name":"Computer science","score":0.7585617899894714},{"id":"https:\/\/openalex.org\/keywords\/metric","display_name":"Metric (unit)","score":0.7187696695327759},{"id":"https:\/\/openalex.org\/keywords\/task","display_name":"Task (project management)","score":0.6305834054946899},{"id":"https:\/\/openalex.org\/keywords\/artificial-intelligence","display_name":"Artificial intelligence","score":0.6220201849937439},{"id":"https:\/\/openalex.org\/keywords\/natural-language-generation","display_name":"Natural language generation","score":0.6045323014259338},{"id":"https:\/\/openalex.org\/keywords\/sentence","display_name":"Sentence","score":0.5598405003547668},{"id":"https:\/\/openalex.org\/keywords\/machine-learning","display_name":"Machine learning","score":0.5346941351890564},{"id":"https:\/\/openalex.org\/keywords\/natural-language-processing","display_name":"Natural language processing","score":0.5178306698799133},{"id":"https:\/\/openalex.org\/keywords\/quality","display_name":"Quality (philosophy)","score":0.473473459482193},{"id":"https:\/\/openalex.org\/keywords\/dimension","display_name":"Dimension (graph theory)","score":0.4693295359611511},{"id":"https:\/\/openalex.org\/keywords\/face","display_name":"Face (sociological concept)","score":0.4685821235179901},{"id":"https:\/\/openalex.org\/keywords\/measure","display_name":"Measure (data warehouse)","score":0.4119381606578827},{"id":"https:\/\/openalex.org\/keywords\/natural-language","display_name":"Natural language","score":0.2926839590072632},{"id":"https:\/\/openalex.org\/keywords\/data-mining","display_name":"Data mining","score":0.20364582538604736},{"id":"https:\/\/openalex.org\/keywords\/linguistics","display_name":"Linguistics","score":0.08866885304450989},{"id":"https:\/\/openalex.org\/keywords\/mathematics","display_name":"Mathematics","score":0.0824311375617981}],"concepts":[{"id":"https:\/\/openalex.org\/C2781067378","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q17027399","display_name":"Interpretability","level":2,"score":0.9209123849868774},{"id":"https:\/\/openalex.org\/C170858558","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8266123533248901},{"id":"https:\/\/openalex.org\/C177148314","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q170084","display_name":"Generalization","level":2,"score":0.8227543234825134},{"id":"https:\/\/openalex.org\/C41008148","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q21198","display_name":"Computer science","level":0,"score":0.7585617899894714},{"id":"https:\/\/openalex.org\/C176217482","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q860554","display_name":"Metric (unit)","level":2,"score":0.7187696695327759},{"id":"https:\/\/openalex.org\/C2780451532","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q759676","display_name":"Task (project management)","level":2,"score":0.6305834054946899},{"id":"https:\/\/openalex.org\/C154945302","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6220201849937439},{"id":"https:\/\/openalex.org\/C2776187449","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1513879","display_name":"Natural language generation","level":3,"score":0.6045323014259338},{"id":"https:\/\/openalex.org\/C2777530160","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q41796","display_name":"Sentence","level":2,"score":0.5598405003547668},{"id":"https:\/\/openalex.org\/C119857082","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q2539","display_name":"Machine learning","level":1,"score":0.5346941351890564},{"id":"https:\/\/openalex.org\/C204321447","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q30642","display_name":"Natural language processing","level":1,"score":0.5178306698799133},{"id":"https:\/\/openalex.org\/C2779530757","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.473473459482193},{"id":"https:\/\/openalex.org\/C33676613","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.4693295359611511},{"id":"https:\/\/openalex.org\/C2779304628","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4685821235179901},{"id":"https:\/\/openalex.org\/C2780009758","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4119381606578827},{"id":"https:\/\/openalex.org\/C195324797","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q33742","display_name":"Natural language","level":2,"score":0.2926839590072632},{"id":"https:\/\/openalex.org\/C124101348","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q172491","display_name":"Data mining","level":1,"score":0.20364582538604736},{"id":"https:\/\/openalex.org\/C41895202","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q8162","display_name":"Linguistics","level":1,"score":0.08866885304450989},{"id":"https:\/\/openalex.org\/C33923547","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q395","display_name":"Mathematics","level":0,"score":0.0824311375617981},{"id":"https:\/\/openalex.org\/C187736073","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q2920921","display_name":"Management","level":1,"score":0},{"id":"https:\/\/openalex.org\/C138885662","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q5891","display_name":"Philosophy","level":0,"score":0},{"id":"https:\/\/openalex.org\/C111472728","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q9471","display_name":"Epistemology","level":1,"score":0},{"id":"https:\/\/openalex.org\/C21547014","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1423657","display_name":"Operations management","level":1,"score":0},{"id":"https:\/\/openalex.org\/C202444582","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q837863","display_name":"Pure mathematics","level":1,"score":0},{"id":"https:\/\/openalex.org\/C134306372","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q7754","display_name":"Mathematical analysis","level":1,"score":0},{"id":"https:\/\/openalex.org\/C162324750","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q8134","display_name":"Economics","level":0,"score":0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2307.06869","is_oa":true,"landing_page_url":"http:\/\/arxiv.org\/abs\/2307.06869","pdf_url":"https:\/\/arxiv.org\/pdf\/2307.06869","source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550\/arxiv.2307.06869","is_oa":true,"landing_page_url":"https:\/\/doi.org\/10.48550\/arxiv.2307.06869","pdf_url":null,"source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.06869","is_oa":true,"landing_page_url":"http:\/\/arxiv.org\/abs\/2307.06869","pdf_url":"https:\/\/arxiv.org\/pdf\/2307.06869","source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https:\/\/metadata.un.org\/sdg\/4","score":0.8399999737739563}],"awards":[{"id":"https:\/\/openalex.org\/G7155907028","display_name":null,"funder_award_id":"61936010","funder_id":"https:\/\/openalex.org\/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https:\/\/openalex.org\/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https:\/\/ror.org\/01h0zpd94"},{"id":"https:\/\/openalex.org\/F4320322392","display_name":"Tsinghua University","ror":"https:\/\/ror.org\/03cve4549"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https:\/\/content.openalex.org\/works\/W4384390283.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https:\/\/openalex.org\/W2905433371","https:\/\/openalex.org\/W2888392564","https:\/\/openalex.org\/W4310278675","https:\/\/openalex.org\/W4388422664","https:\/\/openalex.org\/W4390569940","https:\/\/openalex.org\/W4361193272","https:\/\/openalex.org\/W2366403280","https:\/\/openalex.org\/W4389519239","https:\/\/openalex.org\/W2050523636","https:\/\/openalex.org\/W4323651265"],"abstract_inverted_index":{"Existing":[0],"evaluation":[1,28,35,50,84,101,112,154],"metrics":[2,22,46,166],"for":[3,52,167],"natural":[4],"language":[5,95],"generation":[6],"(NLG)":[7],"tasks":[8,33],"face":[9],"the":[10,20,57,106,111,123,129,133,153],"challenges":[11],"on":[12,27,100],"generalization":[13,107,181],"ability":[14,182],"and":[15,34,91,171,183],"interpretability.":[16,184],"Specifically,":[17],"most":[18],"of":[19,30,125,135],"well-performed":[21],"are":[23,146],"required":[24],"to":[25,41,59,104,151],"train":[26],"datasets":[29],"specific":[31],"NLG":[32,83],"dimensions,":[36],"which":[37,174],"may":[38],"cause":[39],"over-fitting":[40],"task-specific":[42],"datasets.":[43],"Furthermore,":[44],"existing":[45],"only":[47],"provide":[48],"an":[49,86],"score":[51,63],"each":[53,136],"dimension":[54],"without":[55,98],"revealing":[56],"evidence":[58,150],"interpret":[60],"how":[61],"this":[62],"is":[64],"obtained.":[65],"To":[66,109],"deal":[67],"with":[68,140],"these":[69],"challenges,":[70],"we":[71,116],"propose":[72],"a":[73],"simple":[74],"yet":[75],"effective":[76],"metric":[77,81],"called":[78],"DecompEval.":[79],"This":[80],"formulates":[82],"as":[85,149],"instruction-style":[87,120],"question":[88,121],"answering":[89],"task":[90],"utilizes":[92],"instruction-tuned":[93],"pre-trained":[94],"models":[96],"(PLMs)":[97],"training":[99],"datasets,":[102],"aiming":[103],"enhance":[105],"ability.":[108],"make":[110],"process":[113],"more":[114],"interpretable,":[115],"decompose":[117],"our":[118],"devised":[119],"about":[122],"quality":[124,134],"generated":[126,143],"texts":[127],"into":[128],"subquestions":[130,139],"that":[131,159],"measure":[132],"sentence.":[137],"The":[138],"their":[141],"answers":[142],"by":[144],"PLMs":[145],"then":[147],"recomposed":[148],"obtain":[152],"result.":[155],"Experimental":[156],"results":[157],"show":[158],"DecompEval":[160],"achieves":[161],"state-of-the-art":[162],"performance":[163],"in":[164],"untrained":[165],"evaluating":[168],"text":[169],"summarization":[170],"dialogue":[172],"generation,":[173],"also":[175],"exhibits":[176],"strong":[177],"dimension-level":[178],"\/":[179],"task-level":[180]},"counts_by_year":[],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2023-07-15T00:00:00"}