{"id":"https:\/\/openalex.org\/W4381713211","doi":"https:\/\/doi.org\/10.48550\/arxiv.2306.11943","title":"Towards Understanding What Code Language Models Learned","display_name":"Towards Understanding What Code Language Models Learned","publication_year":2023,"publication_date":"2023-06-20","ids":{"openalex":"https:\/\/openalex.org\/W4381713211","doi":"https:\/\/doi.org\/10.48550\/arxiv.2306.11943"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2306.11943","is_oa":true,"landing_page_url":"http:\/\/arxiv.org\/abs\/2306.11943","pdf_url":"https:\/\/arxiv.org\/pdf\/2306.11943","source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https:\/\/arxiv.org\/pdf\/2306.11943","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https:\/\/openalex.org\/A5072573553","display_name":"Toufique Ahmed","orcid":"https:\/\/orcid.org\/0000-0002-4427-1350"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ahmed, Toufique","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5113226487","display_name":"Dian Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Dian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5112943666","display_name":"Chengxuan Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Chengxuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5054914260","display_name":"Cathy Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Cathy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https:\/\/openalex.org\/A5036744986","display_name":"Pr\u00e9mkumar D\u00e9vanbu","orcid":"https:\/\/orcid.org\/0000-0002-4346-5276"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Devanbu, Prem","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https:\/\/openalex.org\/A5070742827","display_name":"Kenji Sagae","orcid":"https:\/\/orcid.org\/0000-0003-3371-0618"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sagae, Kenji","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https:\/\/openalex.org\/A5072573553"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https:\/\/openalex.org\/T10028","display_name":"Topic Modeling","score":0.9918000102043152,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1702","display_name":"Artificial Intelligence"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}},"topics":[{"id":"https:\/\/openalex.org\/T10028","display_name":"Topic Modeling","score":0.9918000102043152,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1702","display_name":"Artificial Intelligence"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}},{"id":"https:\/\/openalex.org\/T10181","display_name":"Natural Language Processing Techniques","score":0.9883999824523926,"subfield":{"id":"https:\/\/openalex.org\/subfields\/1702","display_name":"Artificial Intelligence"},"field":{"id":"https:\/\/openalex.org\/fields\/17","display_name":"Computer Science"},"domain":{"id":"https:\/\/openalex.org\/domains\/3","display_name":"Physical Sciences"}},{"id":"https:\/\/openalex.org\/T12090","display_name":"Language and cultural evolution","score":0.9573000073432922,"subfield":{"id":"https:\/\/openalex.org\/subfields\/3316","display_name":"Cultural Studies"},"field":{"id":"https:\/\/openalex.org\/fields\/33","display_name":"Social Sciences"},"domain":{"id":"https:\/\/openalex.org\/domains\/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https:\/\/openalex.org\/keywords\/computer-science","display_name":"Computer science","score":0.7906766533851624},{"id":"https:\/\/openalex.org\/keywords\/code","display_name":"Code (set theory)","score":0.648268461227417},{"id":"https:\/\/openalex.org\/keywords\/semantics","display_name":"Semantics (computer science)","score":0.6364996433258057},{"id":"https:\/\/openalex.org\/keywords\/meaning","display_name":"Meaning (existential)","score":0.5908262133598328},{"id":"https:\/\/openalex.org\/keywords\/natural-language-processing","display_name":"Natural language processing","score":0.5825622081756592},{"id":"https:\/\/openalex.org\/keywords\/representation","display_name":"Representation (politics)","score":0.5727596282958984},{"id":"https:\/\/openalex.org\/keywords\/variety","display_name":"Variety (cybernetics)","score":0.557674765586853},{"id":"https:\/\/openalex.org\/keywords\/artificial-intelligence","display_name":"Artificial intelligence","score":0.5474463701248169},{"id":"https:\/\/openalex.org\/keywords\/natural-language","display_name":"Natural language","score":0.5429998636245728},{"id":"https:\/\/openalex.org\/keywords\/programming-language","display_name":"Programming language","score":0.4910769462585449},{"id":"https:\/\/openalex.org\/keywords\/contrast","display_name":"Contrast (vision)","score":0.41168129444122314},{"id":"https:\/\/openalex.org\/keywords\/psychology","display_name":"Psychology","score":0.09940171241760254}],"concepts":[{"id":"https:\/\/openalex.org\/C41008148","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q21198","display_name":"Computer science","level":0,"score":0.7906766533851624},{"id":"https:\/\/openalex.org\/C2776760102","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q5139990","display_name":"Code (set theory)","level":3,"score":0.648268461227417},{"id":"https:\/\/openalex.org\/C184337299","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.6364996433258057},{"id":"https:\/\/openalex.org\/C2780876879","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.5908262133598328},{"id":"https:\/\/openalex.org\/C204321447","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q30642","display_name":"Natural language processing","level":1,"score":0.5825622081756592},{"id":"https:\/\/openalex.org\/C2776359362","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5727596282958984},{"id":"https:\/\/openalex.org\/C136197465","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.557674765586853},{"id":"https:\/\/openalex.org\/C154945302","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5474463701248169},{"id":"https:\/\/openalex.org\/C195324797","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q33742","display_name":"Natural language","level":2,"score":0.5429998636245728},{"id":"https:\/\/openalex.org\/C199360897","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q9143","display_name":"Programming language","level":1,"score":0.4910769462585449},{"id":"https:\/\/openalex.org\/C2776502983","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q690182","display_name":"Contrast (vision)","level":2,"score":0.41168129444122314},{"id":"https:\/\/openalex.org\/C15744967","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q9418","display_name":"Psychology","level":0,"score":0.09940171241760254},{"id":"https:\/\/openalex.org\/C17744445","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q36442","display_name":"Political science","level":0,"score":0},{"id":"https:\/\/openalex.org\/C542102704","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q183257","display_name":"Psychotherapist","level":1,"score":0},{"id":"https:\/\/openalex.org\/C199539241","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q7748","display_name":"Law","level":1,"score":0},{"id":"https:\/\/openalex.org\/C177264268","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0},{"id":"https:\/\/openalex.org\/C94625758","wikidata":"https:\/\/www.wikidata.org\/wiki\/Q7163","display_name":"Politics","level":2,"score":0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2306.11943","is_oa":true,"landing_page_url":"http:\/\/arxiv.org\/abs\/2306.11943","pdf_url":"https:\/\/arxiv.org\/pdf\/2306.11943","source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550\/arxiv.2306.11943","is_oa":true,"landing_page_url":"https:\/\/doi.org\/10.48550\/arxiv.2306.11943","pdf_url":null,"source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2306.11943","is_oa":true,"landing_page_url":"http:\/\/arxiv.org\/abs\/2306.11943","pdf_url":"https:\/\/arxiv.org\/pdf\/2306.11943","source":{"id":"https:\/\/openalex.org\/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https:\/\/openalex.org\/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https:\/\/openalex.org\/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"score":0.8399999737739563,"id":"https:\/\/metadata.un.org\/sdg\/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https:\/\/content.openalex.org\/works\/W4381713211.pdf","grobid_xml":"https:\/\/content.openalex.org\/works\/W4381713211.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https:\/\/openalex.org\/W2032233321","https:\/\/openalex.org\/W3121970507","https:\/\/openalex.org\/W2110028391","https:\/\/openalex.org\/W54497855","https:\/\/openalex.org\/W217960748","https:\/\/openalex.org\/W3125814499","https:\/\/openalex.org\/W2090827041","https:\/\/openalex.org\/W2094012830","https:\/\/openalex.org\/W187246281","https:\/\/openalex.org\/W2079194830"],"abstract_inverted_index":{"Pre-trained":[0],"language":[1,10,34],"models":[2,35,63,70,95,120],"are":[3],"effective":[4],"in":[5,71],"a":[6,72,82,124],"variety":[7],"of":[8,21,40,49,81,99,112,121,127,131,138],"natural":[9],"tasks,":[11],"but":[12],"it":[13],"has":[14],"been":[15],"argued":[16],"their":[17,44],"capabilities":[18],"fall":[19],"short":[20],"fully":[22],"learning":[23],"meaning":[24],"or":[25],"understanding":[26],"language.":[27],"To":[28],"understand":[29],"the":[30,97,110,128],"extent":[31],"to":[32,46,58,85],"which":[33,101],"can":[36],"learn":[37,86,123],"some":[38],"form":[39,139],"meaning,":[41],"we":[42,67,91,115],"investigate":[43],"ability":[45,84],"capture":[47,96],"semantics":[48,98,130],"code":[50,113,118,122,132],"beyond":[51,135],"superficial":[52,136],"frequency":[53],"and":[54,78,104],"co-occurrence.":[55],"In":[56,88],"contrast":[57],"previous":[59],"research":[60],"on":[61],"probing":[62],"for":[64,76],"linguistic":[65],"features,":[66],"study":[68],"pre-trained":[69,119],"setting":[73],"that":[74,117,133],"allows":[75],"objective":[77],"straightforward":[79],"evaluation":[80],"model's":[83],"semantics.":[87],"this":[89],"paper,":[90],"examine":[92],"whether":[93],"such":[94],"code,":[100],"is":[102],"precisely":[103],"formally":[105],"defined.":[106],"Through":[107],"experiments":[108],"involving":[109],"manipulation":[111],"fragments,":[114],"show":[116],"robust":[125],"representation":[126],"computational":[129],"goes":[134],"features":[137],"alone":[140]},"counts_by_year":[{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-12T08:34:05.389933","created_date":"2025-10-10T00:00:00"}