diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c25cc7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*~ +log_*.txt +dist/ +build/ +logs/ +work/ +*.log +*.graphml +*.txt +*.zip diff --git a/README.md b/README.md index e2a9ea3..d35674a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ coref ===== -coref \ No newline at end of file +Process a graph containing named entities and documents to group like-named entities and find coreferences within +and across documents. + +Mostly Blueprints gremlin groovy code and a few scripts. diff --git a/build.xml b/build.xml new file mode 100644 index 0000000..c480553 --- /dev/null +++ b/build.xml @@ -0,0 +1,167 @@ + + + + Build Groovy vizlinc-ingester jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data/ner-model.ser.gz b/data/ner-model.ser.gz new file mode 100755 index 0000000..4e0d2ff Binary files /dev/null and b/data/ner-model.ser.gz differ diff --git a/data/splash-ingester.bmp b/data/splash-ingester.bmp new file mode 100644 index 0000000..59e53cb Binary files /dev/null and b/data/splash-ingester.bmp differ diff --git a/documentation/Building the VizLinc Ingester.docx b/documentation/Building the VizLinc Ingester.docx new file mode 100644 index 0000000..5349691 Binary files /dev/null and b/documentation/Building the VizLinc Ingester.docx differ diff --git a/documentation/Building the VizLinc Ingester.pdf b/documentation/Building the VizLinc Ingester.pdf new file mode 100644 index 0000000..87f3fd8 Binary files /dev/null and b/documentation/Building the VizLinc Ingester.pdf differ diff --git a/documentation/Installing and Running the VizLinc Ingester - English.docx b/documentation/Installing and Running the VizLinc Ingester - English.docx new file mode 100644 index 0000000..1435163 Binary files /dev/null and b/documentation/Installing and Running the VizLinc Ingester - English.docx differ diff --git a/documentation/Installing and Running the VizLinc Ingester - English.pdf b/documentation/Installing and Running the VizLinc Ingester - English.pdf new file mode 100644 index 0000000..7b0c902 Binary files /dev/null and b/documentation/Installing and Running the VizLinc Ingester - English.pdf differ diff --git a/lib/commons-io-2.4.jar b/lib/commons-io-2.4.jar new file mode 100644 index 0000000..90035a4 Binary files /dev/null and b/lib/commons-io-2.4.jar differ diff --git a/lib/postgresql-9.2-1002.jdbc4.jar b/lib/postgresql-9.2-1002.jdbc4.jar new file mode 100644 index 0000000..abd06ba Binary files /dev/null and b/lib/postgresql-9.2-1002.jdbc4.jar differ diff --git a/lib/secondstring-20120620.jar b/lib/secondstring-20120620.jar new file mode 100644 index 0000000..d08e547 Binary files /dev/null and b/lib/secondstring-20120620.jar differ diff --git a/lib/stanford-ner-2014-01-04.jar b/lib/stanford-ner-2014-01-04.jar new file mode 100644 index 0000000..b6b53e2 Binary files /dev/null and b/lib/stanford-ner-2014-01-04.jar differ diff --git a/lib/stax2-api-3.1.1.jar b/lib/stax2-api-3.1.1.jar new file mode 100644 index 0000000..63a8a06 Binary files /dev/null and b/lib/stax2-api-3.1.1.jar differ diff --git a/lib/super-csv-2.1.0.jar b/lib/super-csv-2.1.0.jar new file mode 100644 index 0000000..6a85716 Binary files /dev/null and b/lib/super-csv-2.1.0.jar differ diff --git a/lib/tika-app-1.4.jar b/lib/tika-app-1.4.jar new file mode 100644 index 0000000..c204d7a Binary files /dev/null and b/lib/tika-app-1.4.jar differ diff --git a/lib/woodstox-core-asl-4.2.0.jar b/lib/woodstox-core-asl-4.2.0.jar new file mode 100644 index 0000000..8ca96d5 Binary files /dev/null and b/lib/woodstox-core-asl-4.2.0.jar differ diff --git a/nyt-2007_small/1815916.xml.txt b/nyt-2007_small/1815916.xml.txt new file mode 100644 index 0000000..a83f54c --- /dev/null +++ b/nyt-2007_small/1815916.xml.txt @@ -0,0 +1,32 @@ +Jeb Bush Ponders Future, Not Knowing What It Holds + + Jeb Bush Ponders Future, Not Knowing What It Holds + +When the same old irksome question popped up recently at one of his final public events here, Gov. Jeb Bush, addressing Spanish-speaking reporters, gave an atypically dramatic answer: ''Yo no tengo futuro,'' or ''I have no future.'' + His words set off round-the-world buzz, with The Daily Telegraph of London going so far as to call them ''a recognition by the Bush family that their dynastic reign in American politics is drawing to a close.'' + But in fact, the question lives on. Mr. Bush's spokeswoman said last week that he made the comment jokingly, and when asked about it later in an e-mail message, Mr. Bush himself replied, ''I was misunderstood by a reporter.'' + He did not elaborate, leaving the world to know only this much: Half his life after he arrived in Miami as a 27-year-old real estate salesman, Governor Bush returns here this week without the title before his name and, he insists, without knowing what his future holds. + ''We're in the preface of the new book in my life and I just don't know yet,'' he told reporters last month in Tallahassee, a day after his official portrait, with a Bible and a BlackBerry in the background, was unveiled at the Governor's Mansion. ''I'm going to take some time off, hopefully do a little fishing, golfing, resting, reading, exercising. And I've got to make a living, so I'll figure it out probably in January.'' + Florida, too, has some readjusting to do. After eight years in office, Mr. Bush, 53, is leaving as one of the most popular and prominent governors in state history, not least because of his relationship to President Bush (brother) and former President George Bush (son). Succeeding him is Attorney General Charlie Crist, who is Republican like Mr. Bush but otherwise starkly different. + Despite the wishful prodding of admirers, Mr. Bush has adamantly ruled out a presidential campaign of his own next year, saying that he wants only to return to Miami with his wife, Columba, and their cat, Sugar. Yet rumors about his future have burst forth as regularly as exotic species in the Everglades -- among them that he would be the next commissioner of the National Football League, run for Senate or become Senator John McCain's running mate if Mr. McCain won the Republican nomination for president in 2008. + ''The presidency is out of the question at this point because of Bush fatigue,'' said Peter Schweizer, a fellow at the Hoover Institution at Stanford who wrote ''The Bushes: A Dynasty'' with his wife, Rochelle. ''But the vice presidential slot is something that's very much in play. He's a successful governor of an important state, he helps shore up relations with the social conservatives and he has the Bush money machine.'' + One of Mr. Bush's former chiefs of staff has gone to work for Mr. McCain's exploratory committee, but several other former aides have signed up with Gov. Mitt Romney of Massachusetts, another probable Republican contender. + ''Jeb is a policy-driven guy,'' Mr. Schweizer said. ''If he can be a vice president that plays some kind of a policy role as Cheney has, as Gore did in the Clinton administration, then Jeb Bush will be interested.'' + Many assume that for now -- at least partly at the urging of his wife, described as shy and eager to be out of the public eye -- Mr. Bush will return to the private sector. He reported a net worth of $1.4 million in 2005, down from $2.4 million in 1998. + He was a partner in a major real estate development firm here until his first, unsuccessful run for governor in 1994, but Mr. Schweizer predicted that Mr. Bush might now seek out work involving the bioscience industry or the Latin American economy, both of which ''he seems particularly animated by.'' + All indications notwithstanding, ardent admirers like Grover Norquist, the president of Americans for Tax Reform, are not giving up on the prospect of Mr. Bush jumping into the presidential race next year, especially if Senator Hillary Rodham Clinton of New York becomes the Democratic candidate. + ''He could step in later than anybody else,'' Mr. Norquist said. ''You can run for president with the last name of Bush, even though there is and will be Bush fatigue, in a year that you're likely to be running against someone whose last name is Clinton.'' + For the time being, Mr. Bush bought a car, a Chrysler 300C, and rented a $5,500-per-month, 3,949-square-foot condominium in Segovia Tower, a luxury building overlooking a golf course in lush Coral Gables. + ''I have no idea what I will be doing next,'' he wrote by e-mail from Boca Chica, Fla., where he was vacationing with his parents. ''My priorities are to hang out with my beloved wife (until she can't take it anymore! :)), work out every day and figure out what I will do next with my life.'' + As for the continued speculation, he wrote: ''I am flattered that all sorts of people are interested in what I am going to do and many have offered advice as well. That will all subside soon.'' + Small signs suggest, however, that he will have a hard time giving up executive powers. He told reporters that while buying furniture recently, he had to stifle the urge to tell the store owner a better way of doing business -- a trait his adversaries say they will not miss. + ''Bush was the type that if you did not agree with him, he really didn't have time for you,'' said State Senator Frederica Wilson, Democrat of Miami. ''He wanted you to rubber stamp every idea he had, and he wouldn't listen to reason.'' + While Mr. Bush is internationally famous, Mr. Crist, who will be sworn in as governor on Tuesday, is a stranger to all outside Florida and, but for his native Tampa Bay region, not particularly well known within the state either. While Mr. Bush was ideologically driven, often making enemies in pursuit of ''big, hairy, audacious goals'' and divisive social policies, Mr. Crist seems above all a pleaser, avoiding firm opinions and promising to be ''the people's governor.'' + Yet despite Mr. Bush's abrasiveness and the plunging popularity of his brother the president, he has remained well liked -- or at least respected -- to the end, a feat in a state as ethnically and politically divided as Florida. A poll last month by Quinnipiac University found that 57 percent of Floridians feel he did a ''good'' or ''great'' job as governor, compared with only 10 percent who said he had done a ''bad'' job. + Howard Simon, executive director of the American Civil Liberties Union of Florida, said the poll results reflected approval of Mr. Bush's persona more than of his policies. Mr. Simon pointed out that two major education initiatives during the governor's tenure -- a costly effort to lower class size and another to provide universal prekindergarten classes -- were passed by public referendum, over the governor's objections. + ''It needs to be said that the personal appeal and likeability of Jeb Bush has led the press and the public to overlook the extremism of many of his policies,'' he said. + Several of Mr. Bush's pet initiatives in fact failed, including a school voucher program that the Florida Supreme Court found unconstitutional. + But Mr. Bush pushed through $19.3 billion in tax cuts, put an unprecedented emphasis on standardized testing in public schools, privatized thousands of government jobs and ended affirmative action in public university admissions. He also persuaded the Scripps Research Institute and other bioscience research groups to open laboratories in Florida, which he says will makethe state economy less dependent on tourism and create more high-paying jobs. + And he has appointed more than a third of the state's judges, assuring that his socially and fiscally conservative beliefs will continue to hold some sway. + While others have emoted about Mr. Bush's departure -- including his father, who wept as he described his second son's ''decency'' and ''honor'' in a speech in Tallahassee last month -- he has characteristically avoided introspection. Asked last month what he would miss most about the Governor's Mansion, he cited its beauty, its staff -- and its towels. + ''Fresh towels -- all you want,'' he said. ''Here, although I've been trained to do otherwise, it's just any time I want I can have many towels.'' \ No newline at end of file diff --git a/nyt-2007_small/1818955.xml.txt b/nyt-2007_small/1818955.xml.txt new file mode 100644 index 0000000..9cd08b4 --- /dev/null +++ b/nyt-2007_small/1818955.xml.txt @@ -0,0 +1,35 @@ +Bush Proposal For Iraq Adds To '08 Intrigue + + Bush Plan for Iraq Swells ’08 Intrigue + +On Friday morning, as the capital was enmeshed in a debate over President Bush's decision to send thousands more troops to Iraq, no one was more obviously searching for an alternate plan than Senator Hillary Rodham Clinton, the New York Democrat and likely candidate for president. + Mrs. Clinton was airborne for Iraq, a visit that offered a platform to examine and perhaps alter her views on a war that she once voted to authorize but now criticizes. Asked in a telephone interview from Kuwait whether this trip might lead her to support setting a firm deadline for the withdrawal of all American troops, which would be a change in her position, she responded: ''Those are questions that we'll be better able to answer later. Obviously I'll have more to say when I get back.'' + Around the same time in Washington, Senator John McCain, the Arizona Republican and likely candidate for president, threw the White House something of a lifeline during a Congressional hearing (which Mrs. Clinton herself was supposed to attend), vigorously defending Mr. Bush's decision. + ''Potentially catastrophic consequences of failure demand that we do all we can to prevail in Iraq,'' Mr. McCain said. + As the politics of the war continue to shift, no prospective presidential candidates face more intensive scrutiny of their views or greater risk to their ambitions than Mr. McCain and Mrs. Clinton. + Long a proponent of sending more American forces into the fight, Mr. McCain now more than ever is linked to a war that has splintered his party and lost public support. + Viewed with suspicion by elements of her party because she has not explicitly repudiated her October 2002 vote authorizing military action against Iraq, Mrs. Clinton, who was in Baghdad on Saturday, has been trying to shore up her credentials as a critic of the war without exposing herself to charges of indecisiveness or flip-flopping of the sort that so damaged Senator John Kerry in his unsuccessful race against Mr. Bush in 2004. + More than any other single event in recent months, Mr. Bush's call for more troops has not only illuminated the positions taken by Mr. McCain and Mrs. Clinton and the differences between them, but it has also led the field of declared and prospective candidates circling them within their own parties to stake out more clearly defined positions on the war. + Among Republicans, a result has been a deep split suggesting that Mr. McCain will face challenges on the war from fellow Republicans long before he has to worry about Mrs. Clinton and other Democrats. Mitt Romney, the former governor of Massachusetts who had until this week declined to say whether he supported more troops, issued brief remarks supporting the president. Rudolph W. Giuliani, the former New York mayor, also supported it, comparing it to what he did in fighting crime as mayor. + But Senator Sam Brownback of Kansas, who is positioning himself as an alternative to Mr. McCain among conservatives, denounced Mr. Bush's plan, as did Senator Chuck Hagel of Nebraska, who like Mr. McCain is a Vietnam veteran but who long ago broke with the White House over its handling of the war. + Among Democrats, opposition to Mr. Bush's plan highlighted the political threat Mrs. Clinton faces from the left. John Edwards, the former senator from North Carolina who voted for the war in 2002 but who, unlike Mrs. Clinton, later renounced his vote, sent an e-mail message to his supporters urging them to oppose Mr. Bush's action. + Senator Barack Obama of Illinois, who joined the Senate in 2005 and thus escaped the Iraq vote that has come to haunt Mrs. Clinton, Mr. Edwards and Mr. Kerry, used the platform of Senate hearings to lacerate the Bush Iraq policy and affirm his own opposition to the war. + Their actions offered an obvious contrast with those of Mrs. Clinton, who put out a measured statement opposing the troop increase but has stayed away from the Senate vanguard opposing Mr. Bush's latest strategy. + Still, none of the other potential 2008 presidential candidates have seen their ambitions complicated by the events this week as much as Mr. McCain and Mrs. Clinton have. Their responses to Mr. Bush's actions offered insights into their views on Iraq and the styles they would bring to a presidential campaign. + Mr. McCain embarked on a high-profile television tour announcing his support for Mr. Bush's move. In an interview, he said he would have preferred that the White House send in even more troops, and noted that he had pressed this position on the White House, unsuccessfully until now, for more than two years. It is a position that Mr. McCain and his advisers -- and some other Republicans -- said could end up destroying his presidential ambitions. + ''Friends, advisers, enemies said this could be very damaging,'' Mr. McCain said. ''I can't worry about that, and I'm not going to worry about that.'' + Some Republicans suggested that Mr. McCain's position could help him, albeit to a limited extent, among conservative Republican primary voters who have been wary of his view on social issues. And, as Mr. McCain's own remarks suggested, his stance fits neatly into his effort over the years to present himself as a politician given to candor even if it means telling voters things they do not want to hear. + Still, by any measure, support for the war could make things extremely difficult for Mr. McCain in a general election if the situation in Iraq does not improve and public opinion in the United States does not shift in favor of continued engagement. + Mr. McCain did not dispute the political risks entailed in the stance he had taken. + ''The consequences of failure are more important than a political career,'' he said. ''It could hurt me in a primary and it could hurt me in a general election. This could hurt me in history.'' + He said he was hopeful that this new Iraq strategy would work but did not know how long it would take, a crucial question in trying to gauge how central an issue this will be as the campaign engages. + In a hint of what next year could be like, Mr. Edwards has taken to calling the Bush proposal an endorsement of ''the McCain doctrine,'' a description that Mr. McCain and his aides have gamely embraced. + ''I was glad to see that Romney and Giuliani have come around to embracing the McCain doctrine, as John Edwards would say,'' said Terry Nelson, the head of Mr. McCain's presidential exploratory committee. + If Mr. McCain has flaunted his unpopular position, Mrs. Clinton has clearly kept a lower altitude as she tries to deal with a barrage of conflicting pressures. Unlike Mr. McCain and Mr. Obama, who were everywhere on television the night of Mr. Bush's speech, Mrs. Clinton simply issued a late-evening e-mail statement saying that based on what Mr. Bush had said, she did not support the increase. + Last year, Mrs. Clinton took a leading role in criticizing Donald H. Rumsfeld, then the secretary of defense, over his management of the war. Over the past two years, she has voted in favor of resolutions calling for a phased withdrawal of troops without setting fixed deadlines. + Even though Mrs. Clinton has never gone as far as Mr. Edwards and Mr. Kerry with a renunciation of her 2002 vote, she has subtly expanded her position recently in recognition, her advisers said, of how much that vote accounted for the resistance to her among some Democrats. + In a little-noticed interview on the ''Today'' show just before Christmas, Mrs. Clinton made a significant addition to her standard remark about the 2002 vote. Asked if she considered the vote a mistake, she responded: ''Obviously if we knew then what we know now, there wouldn't have been a vote, and I certainly wouldn't have voted that way.'' + The last eight words are new for her. + Some Democrats suggested that Mrs. Clinton's trip this weekend was intended to give her an opportunity to move further away from that vote and against the president's policy. + ''I will be spending time with the Iraqi leaders and sitting down and trying to get my own personal assessment of what their intentions are, what do they intend to do to take responsibility for their own security,'' she said in the interview. ''If they're not going to step up and try to bring some stability and end the violence in Baghdad, I don't care how many troops the president sends -- it's not going to be successful.'' + Mrs. Clinton and Mr. McCain have no doubt learned a lesson from Mr. Kerry's problems in 2004 when he appeared to switch his position on the war: that inconsistency may well prove to be a bigger political sin than taking a position out of step with the electorate. Mrs. Clinton's associates have expressed concern that if she tries to switch her views on the war, reflecting what her advisers said was concern, she would appear weak as she campaigned during a time of war to become the nation's first female president. \ No newline at end of file diff --git a/nyt-2007_small/1820515.xml.txt b/nyt-2007_small/1820515.xml.txt new file mode 100644 index 0000000..25d645b --- /dev/null +++ b/nyt-2007_small/1820515.xml.txt @@ -0,0 +1,13 @@ +Announcement Sends Race Into High Gear + +Senator Hillary Rodham Clinton's entry into the 2008 presidential contest yesterday set off rounds of e-mail messages and conference calls among both her allies and opponents, some of whom were shaking their heads that a major political event was happening early on a Saturday morning. + Advisers to some of her top 2008 rivals -- Senators John McCain and Barack Obama, and former Gov. Mitt Romney of Massachusetts -- replied within minutes to requests for comment, and signaled that the Clinton announcement meant that the 2008 race was truly under way. + ''She's tough enough, smart enough and experienced enough to overcome a decidedly liberal philosophy,'' said John Weaver, a senior adviser to Mr. McCain, who is preparing to seek the Republican presidential nomination. + ''The Clinton, Obama, Edwards chain match will be hard to avert my eyes from, speaking as a pure spectator, of course,'' Mr. Weaver added, referring to former Senator John Edwards of North Carolina. (At least four other Democrats are preparing to campaign as well.) + Clinton advisers said that they chose Jan. 20 as the target date for an announcement during a meeting in mid-December, and that Mrs. Clinton was ''raring to go'' on Friday. + ''I have never been afraid to stand up for what I believe in or to face down the Republican machine,'' Mrs. Clinton said in announcing the formation of a presidential exploratory committee on her Web site, HillaryClinton.com. ''After nearly $70 million spent against my campaigns in New York and two landslide wins, I can say I know how Washington Republicans think, how they operate, and how to beat them.'' + The advisers said she wanted to announce yesterday so that she would dominate the weekend news and Sunday talk shows, and to demonstrate a contrast in leadership with President Bush as he prepares for his State of the Union speech on Tuesday night. + Next weekend, she plans to campaign across Iowa, the site of the first presidential caucuses in January 2008, and visit the early primary state of New Hampshire soon after. + Mrs. Clinton appears at the head of the Democratic pack in many national polls, yet she is in a tighter spot in some voter surveys in Iowa and New Hampshire, which begin the presidential nominating process. Recent polls show Mr. Obama and Mr. Edwards doing well in those states. + ''She has not spent as much time in those state as in others, but this is a woman who is not a stranger for hard work, and who is prepared to go out and work hard for every vote,'' said one Clinton adviser, who spoke on condition of anonymity because he did not want to openly acknowledge that she was running behind to anyone. + Other than Mr. Obama and Mr. Edwards, the 2008 Democratic field at this stage also includes Senators Joseph R. Biden Jr. of Delaware and Christopher J. Dodd of Connecticut; former Gov. Tom Vilsack of Iowa; and Representative Dennis J. Kucinich of Ohio. An eighth possible candidate, Gov. Bill Richardson of New Mexico, was expected to declare today that he is forming an exploratory committee. \ No newline at end of file diff --git a/nyt-2007_small/1820735.xml.txt b/nyt-2007_small/1820735.xml.txt new file mode 100644 index 0000000..fcbdef3 --- /dev/null +++ b/nyt-2007_small/1820735.xml.txt @@ -0,0 +1,33 @@ +RUSH OF ENTRIES GIVES 2008 RACE EARLY INTENSITY + + Rush of Entries Gives ’08 Race Early Intensity + +Two years before the next president is inaugurated and a full year before the first vote is cast, the contest for the White House is off to a breathtakingly fast start, exposing an ever-growing field of candidates to longer, more intensive scrutiny and increasing the amount of money they need to remain viable. + On Sunday, Senator Hillary Rodham Clinton, Democrat of New York, held her first campaign event, highlighting her focus on health care a day after declaring her plans to run. Another Democrat, Gov. Bill Richardson of New Mexico, entered the fray, the eighth member of his party to do so. And the day was not terribly different in its pace of activity from many others in recent weeks. + The scale and swiftness of the action has the potential to upset the traditional timetables and conventions of presidential campaigning. + John Weaver, a senior adviser to Senator John McCain's presidential effort, said the intensified announcement season and compressed primary calendar would force campaigns to develop a strong national apparatus and well-organized field efforts state by state. + ''It makes it nearly impossible for a dark horse candidate to break out of the pack and challenge the front-runner(s) and thus isn't healthy for the process,'' Mr. Weaver wrote in an e-mail message on Sunday. ''All of these states, who are moving up early, want to play and have an impact. But oddly enough, it ultimately will limit the legitimate candidate choices for the nation at large in the primary process.'' + The candidates could be forced to move more quickly to take positions on big issues, stripping them of the chance to run on more gauzy platforms in the early stages and therefore exposing them to more direct criticism from rivals, interest groups and the news media. They will face earlier encounters with one another -- New Hampshire and South Carolina are planning full-scale debates this spring -- that will require them to display both policy expertise and a comfort level in front of the cameras. + They will be getting intensive scrutiny from opposition research operations, the news media and the public for that much longer, increasing the chances that a gaffe or position change could harm their campaigns. Deep into competition for experienced staff members, most candidates are already putting together operations in multiple states. + Kevin Madden, press secretary for the exploratory committee set up by former Gov. Mitt Romney of Massachusetts, a Republican, said his organization was already ''beginning to put our teams together'' for the early contests in New Hampshire, Iowa, South Carolina, Michigan and several states beyond. ''It's happened at a very advanced pace,'' Mr. Madden said, ''but you can't complain and wring your hands. You just have to work harder, faster.'' + Because they do not want competitors to be raising money unchallenged, more candidates are declaring their intentions earlier, which in turn means the entire field needs more money to sustain campaigns for a longer time. + There are now a dozen serious contenders from both parties competing in a presidential race that for the first time in more than half a century will not include an incumbent -- either the president or the vice president -- on the ballot or even a definitive front-runner. + ''Crowded fields force early announcements,'' said Jennifer Palmieri, an adviser to John Edwards, the former senator from North Carolina who is seeking the Democratic nomination. ''Candidates are concerned there will not be enough oxygen left for them if they wait too long. Having crowded fields in both parties has exacerbated this phenomenon.'' + Just hours after Mrs. Clinton made her candidacy official on Saturday, Senator Sam Brownback of Kansas joined the race for the Republican nomination. Last week, Senator Barack Obama, Democrat of Illinois, opened a presidential exploratory committee, emphasizing the wide-open nature of the race. + The early start of the presidential race may make it difficult for the new Democratic leaders in Congress to generate public support and media attention for their agenda. Seven sitting members of the House and Senate have declared their candidacies and several others are said to be considering it, distracting them from legislative business and drawing news coverage away from Congress and out onto the campaign trail. + John D. Podesta, a former chief of staff for President Bill Clinton and president of the Center for American Progress, said some of the early candidates surely recall the lesson of Gen. Wesley K. Clark, the Democrat who waited to jump into the last presidential race until the fall of 2003. + Given the lateness of his entry, and his limited resources, General Clark decided to skip the Iowa caucuses and focus on the New Hampshire primary. Iowa became an unexpectedly fierce contest, with John Kerry emerging as a winner and quickly rolling on to victories in New Hampshire and other early primary states. + ''You need to get a foothold early and organize and get people to rally around you and your message,'' Mr. Podesta said, ''and the need to build momentum is real.'' + The candidates and the early primary states are chasing each other in a mad circle, with two new states, Nevada and South Carolina, squeezing into the first weeks of the primary calendar. A number of other states, including California, New Jersey, Michigan and Illinois, are considering moving up their primaries so that they are not left out of the nominating process. With their expensive media markets, these states could quickly bankrupt candidates who have trouble raising money. + The intensity of the early action is fueled in part by President Bush's political weakness, brought on largely because of the unpopularity of the war in Iraq. + ''If Bush were doing well and had a continuing ability to get things done and command the national stage, I think there would be far less focus on the campaign,'' said Robert Dallek, a presidential historian. + While presidential campaigns have been getting gradually longer over the past few decades, the acceleration in the 2008 cycle is particularly pronounced. The first President Bush announced his candidacy for the 1988 Republican nomination in October 1987; the eventual Democratic nominee in that election, Gov. Michael S. Dukakis of Massachusetts, had declared six months earlier. + Bill Clinton formally announced his candidacy for the 1992 Democratic presidential nomination on Oct. 3, 1991, about three and a half months before the Iowa caucuses. George W. Bush announced his exploratory committee for the 2000 presidential race in March 1999 and began his campaign in June 1999. + By comparison, Mr. Edwards of North Carolina, the 2004 vice-presidential nominee, has traveled to Iowa 16 times since the beginning of last year, building his organization there in hopes of scoring an early triumph that carries him into the next contests. + ''The earlier process will reward candidates who truly have a succinct, credible, authentic and passionate message which can sustain itself over the long nature of the campaign,'' said Matthew Dowd, the chief strategist for Mr. Bush's re-election campaign in 2004. + He also said that in 2007 candidates would be rewarded by scoring points in ''nonvoting events'' such as media attention, their standings in the polls and the size and response of crowds, because those sorts of factors will help winnow the field more than the primaries still a year away. + Despite the intense focus by most candidates on showing that they can raise the money to run a long and expensive campaign, having a big bank account, Mr. Dowd argued, may actually not be as important in the early stages of this presidential cycle as it was in previous ones. + ''It's for two reasons: the early process will not involve paid media as much, and new technology allows little cost to talk directly to voters,'' he said. ''And the early process will make it more important for a campaign to know how to respond to knowable and unknowable events in next 12 months.'' + Steve Elmendorf, a Democratic strategist who worked on the presidential campaigns of Representative Richard A. Gephardt in 2003 and Senator John Kerry in 2004, warned that candidates and their aides, no matter how tired they become, would have to stay on their toes because any misstep might be captured on tape and circulated on the Internet. + ''Every move they make in Iowa and New Hampshire will be on YouTube,'' Mr. Elmendorf said. ''The only certainty by January '08 is that people will be pretty tired.'' + Besides taking a toll on the declared candidates, the length and cost of current campaigns also deters potential entrants. Senator Evan Bayh of Indiana and former Gov. Mark Warner of Virginia were considered among the brightest Democratic prospects, but both declined to run. They cited the crowded field, the endless burden of fund-raising and the brutal personal cost of today's presidential campaigns. \ No newline at end of file diff --git a/nyt-2007_small/1821369.xml.txt b/nyt-2007_small/1821369.xml.txt new file mode 100644 index 0000000..4e1ac76 --- /dev/null +++ b/nyt-2007_small/1821369.xml.txt @@ -0,0 +1,31 @@ +BIG STATES' PUSH FOR EARLIER VOTE SCRAMBLES RACE + + Big States’ Push for Earlier Vote Scrambles Race + +As many as four big states -- California, Florida, Illinois and New Jersey -- are likely to move up their 2008 presidential primaries to early next February, further upending an already unsettled nominating process and forcing candidates of both parties to rethink their campaign strategies, party officials said Wednesday. + The changes, which seem all but certain to be enacted by state legislatures, mean that the presidential candidates face the prospect of going immediately from an ordered series of early contests in relatively small states in January to a single-day, coast-to-coast battlefield in February, encompassing some of the most expensive advertising markets in the nation. + The changes would appear to benefit well-financed and already familiar candidates and diminish the prospects of those with less money and name recognition going into such a highly compressed series of contests early next year. + Associates of Senator Hillary Rodham Clinton, the New York Democrat, and Senator John McCain, the Arizona Republican, said that should either of them stumble early on, the respective party primaries in California and New Jersey -- two states that would seem particularly hospitable to them -- could offer an expensive but welcome firewall. + But at the same time, states like New Jersey and California could provide an opening for Rudolph W. Giuliani, the former mayor of New York, who faces the daunting prospect of overcoming resistance among social conservatives in the Republican contests in Iowa and South Carolina in January. + And several party analysts suggested that having such delegate-rich states at stake on Feb. 5 could persuade candidates who might otherwise step out after a defeat in Iowa or New Hampshire to press on in hopes of a dramatic recovery on the new Super Tuesday. + ''I think this is huge,'' said John Weaver, a senior adviser to Mr. McCain. ''And the unintended consequences could be even bigger.'' + While officials in both parties are wary of the changes, final say over the calendar rests with the states. Advisers to Republican and Democratic presidential candidates say they have come to view substantial changes as inevitable and they have begun to plan accordingly. + ''We don't set the calendar, and we don't control the calendar, but we are going to compete aggressively in all these states,'' said Patti Solis Doyle, who is the manager of Mrs. Clinton's exploratory presidential effort. ''And I will also tell you we have the resources and the organization to compete in all those states.'' + The developments mark the latest upheaval in a political calendar already in disarray. The Democratic Party voted last year to allow Nevada and South Carolina to move their nominating contests into the narrow period at the beginning of the process that once was confined to just Iowa's caucus and New Hampshire's primary. + But New Hampshire officials, protective of their first-in-the-nation primary status, have responded by saying they will schedule their primary as early as it takes, even before Jan. 1, to protect its traditional role. And no one seems to know where the scramble for influence among the states will end. + ''This is completely out of control,'' said William F. Galvin, the Massachusetts secretary of state. He is the leader of a National Association of Secretaries of State committee that is monitoring this movement and trying to push back against it. + ''The issue has been bad,'' Mr. Galvin said. ''But it's never been as bad as it has been this year. In New Hampshire, they are going to be singing Christmas carols and voting.'' + The developments suggest that the national parties are losing any control they have had over the calendar by which they will nominate presidential candidates in 2008. California, New Jersey, Florida and Illinois are most likely to move their primaries early, probably to Feb. 5, joining at least five smaller states that had already scheduled primaries for that day. Illinois lawmakers are talking about moving their primary to help Senator Barack Obama, a Democratic contender; if history is any guide, it is possible that the other candidates might decline to compete in the home state of one of their rivals. + But final votes have not been taken, and state officials said it was possible they could end up going even earlier. Florida in particular has talked about holding its primary seven days after New Hampshire's, at the risk of sanctions from the Democratic National Committee. And officials said that other states, viewing this surge to the front of the pack, could join in as well. + The Democratic National Committee had adopted a new calendar last year, reducing the once dominant influence of Iowa and New Hampshire, after years of consideration. The Republican Party has so far deferred to Democrats on the changes, waiting to see the outcome of the back-and-forth. + Democrats and Republicans said that the changes would be the latest step in the evolution of a presidential nominating system that increasingly seems resistant to the kind of dark-horse presidential bid that was possible back when small states like Iowa and New Hampshire enjoyed such influence over the nominating process. + It has sowed unease and confusion among campaign staff members as they have tried to measure its implications, and has prompted them to begin making moves now to prepare for a whole different nominating system. While conventional wisdom is that the best-known candidates would benefit, views about how the shift might play out vary among strategists. + Tad Devine, a Democratic consultant who was a senior adviser to Democratic presidential candidates in 2004 and 2000, said the calendar changes, combined with the presence on the Democratic side of three strong and well-financed candidates -- Mr. Obama, Mrs. Clinton and, probably, John Edwards, the former North Carolina senator -- could mean that the battle for the nomination drags on for months. + ''I think there's a very good chance that we are going to be sitting here at the end of next March saying, 'How are people going to put together a majority of delegates with 80 percent of the delegates gone?' '' he said. ''The nominating process in 2008 is not a little different. It's fundamentally different.'' + And the campaigns are adjusting accordingly. Mr. McCain recently hired Steve Schmidt, a former Bush campaign operative who managed the re-election bid of Gov. Arnold Schwarzenegger, the California Republican, in no small part because of what he saw as Mr. Schmidt's command of California politics. + Similarly, Mitt Romney, the former governor of Massachusetts, has turned his attention to Florida, where he sees a strong prospect of knocking out Mr. McCain by appealing to social conservatives, and installed a campaign team that includes two top political advisers to former Gov. Jeb Bush of Florida. He is about to announce the hiring of more than six top aides in Florida. + Last weekend, Mr. Romney went to New Jersey, where his aides also think he could make a strong appeal. + ''The focus of our efforts publicly have been in Iowa, South Carolina, Michigan, New Hampshire,'' said Kevin Madden, Mr. Romney's press secretary. ''But we are very well organized in Florida.'' + In the case of Florida, Democrats and Republicans have welcomed the prospect of having to spend heavily on an early primary because that could prove an early investment for the general campaign, as well, considering how competitive that state is. That is not the case in California or New Jersey, two expensive states that have been solidly Democratic in recent presidential races. + And some Democrats disputed the notion that a California victory could help someone recover from a poor showing in the early states. Nick Baldick, a senior adviser to Mr. Edwards, noted that Howard Dean spent heavily in states that held their primaries after New Hampshire and Iowa and never recovered. + ''All that mattered was momentum and winning in Iowa,'' Mr. Baldick said Wednesday. ''I would argue that more states on Feb. 5 makes that exponentially more true. That if you don't have momentum going into states like those four big states, then forget it and just go home.'' \ No newline at end of file diff --git a/nyt-2007_small/1822385.xml.txt b/nyt-2007_small/1822385.xml.txt new file mode 100644 index 0000000..6a31513 --- /dev/null +++ b/nyt-2007_small/1822385.xml.txt @@ -0,0 +1,12 @@ +Ex-Arkansas Governor Takes Step Toward '08 Candidacy + +Former Gov. Mike Huckabee of Arkansas announced Sunday that he was forming a presidential exploratory committee, mounting an underdog campaign for the Republican nomination with ''a message of hope and optimism for restoring America.'' + Mr. Huckabee, 51, is a Baptist minister who served more than 10 years as governor of Arkansas and has close ties with many Christian conservatives. He enters a race largely dominated, so far, by heavyweights like Senator John McCain of Arizona, former Gov. Mitt Romney of Massachusetts and former Mayor Rudolph W. Giuliani of New York. Also, Mr. Huckabee faces stiff competition for the social conservative vote, his natural base, from Senator Sam Brownback of Kansas. + Still, in an interview, Mr. Huckabee was bullish. + ''It's about getting the message to connect with Republican activists and voters, who may not see their guy on the field yet,'' Mr. Huckabee said. He also criticized the emphasis on fund-raising prowess as a measure of political viability. ''It makes me wonder if we should just put the presidency up on eBay,'' he said. + On Sunday, Mr. Huckabee affirmed his support for President Bush's call for an increase in American troops in Iraq. ''He's got to be given the opportunity to do his job,'' Mr. Huckabee said. + In an interview on the NBC news program ''Meet the Press,'' where Mr. Huckabee announced his candidacy, he said that ''to oppose a sitting commander in chief while we've got people being shot at on the ground'' could be ''a very risky thing to our troops.'' + He also spoke about his staunch opposition to abortion rights, saying, ''I always am going to err on the side of life.'' But the anti-abortion movement, he said, has to do ''some growing and expanding.'' + ''We have to remind people that life, where we believe it begins at conception, it doesn't end at birth,'' Mr. Huckabee added. ''And if we're really pro-life, we have to be concerned about more than the gestation period.'' + When asked during the television program if he had ''a problem with gay people,'' Mr. Huckabee replied, ''No, I have a problem with changing institutions that have served us,'' notably marriage. ''Before we change the definition of marriage to mean something different, I think our real focus ought to be on strengthening heterosexual marriages because half of them are ending in divorce,'' he said. + Asked about a recent Democratic entrant to the 2008 field, Senator Hillary Rodham Clinton, Mr. Huckabee said that she would ''absolutely'' be a formidable candidate, and added, ''I think people underestimate her at their own peril.'' \ No newline at end of file diff --git a/nyt-2007_small/1822556.xml.txt b/nyt-2007_small/1822556.xml.txt new file mode 100644 index 0000000..d033605 --- /dev/null +++ b/nyt-2007_small/1822556.xml.txt @@ -0,0 +1,8 @@ +Tethers to the Senate + +The senators eyeing the White House -- there are six or seven, depending on whether one counts Chuck Hagel, Republican of Nebraska -- face a scheduling problem that does not torture their non-Senate rivals. Because of the Senate's close partisan divide, most members are reluctant to miss votes to hit the campaign trail. + So while Mitt Romney, the former Republican governor of Massachusetts, was able to meet voters and donors in South Carolina yesterday and John Edwards, the former Democratic senator from North Carolina, can do the same in New Hampshire and New York tomorrow, members of the Senate presidential caucus will be glued to Washington for a vote on raising the minimum wage. + Next week, it is all hands on the Senate deck for a vote on a resolution disapproving of President Bush's plan to send 21,000 additional troops to Iraq. + That explains why Senator Hillary Rodham Clinton of New York is spending her weekends in Iowa and New Hampshire. Senator Barack Obama of Illinois was in New Orleans yesterday for a hearing on domestic security but planned to return to Washington today. One exception is Senator Sam Brownback, the Kansas Republican. He will be in Iowa, Michigan, South Carolina and Florida today through Friday. A spokesman said he would be in Washington for votes on the Iraq deployment or any other critical questions. + JOHN M. BRODER + POLITICAL ACTION \ No newline at end of file diff --git a/nyt-2007_small/1822756.xml.txt b/nyt-2007_small/1822756.xml.txt new file mode 100644 index 0000000..2f1d7ae --- /dev/null +++ b/nyt-2007_small/1822756.xml.txt @@ -0,0 +1,23 @@ +To '08 Hopefuls, Media Technology Can Be Friend or Foe + + To ’08 Hopefuls, Media Technology Can Be Friend or Foe + +Some of the nation's most enduring memories of Senator Hillary Rodham Clinton -- memories she would happily erase -- were etched on television more than a decade ago: She didn't stay home and bake cookies in her marriage. She wasn't ''some little woman, standing by my man, like Tammy Wynette.'' The headband. The hairstyles. + On Saturday, one week into her presidential campaign, the threat of a new, unflattering image surfaced: MSNBC used a microphone to capture Mrs. Clinton singing the national anthem in Des Moines. Her voice was, shall we say, off key. The recording was quickly downloaded to YouTube, the video-sharing Web site, and the Drudge Report -- no friend of Mrs. Clinton -- was steering readers to watch it. (By Tuesday afternoon, more than 800,000 had.) + Clinton advisers found out about the YouTube video within minutes, and their campaign war room made a calculated decision: not to respond at all. They did not want to draw news media attention to the video; nor did they want to upstage their preferred news of the day, Mrs. Clinton's debut in Iowa. + ''Senator Clinton's candidacy is not premised on her ability to carry a tune,'' said Howard Wolfson, a senior adviser and war room manager. ''We did not see it as a significant threat.'' + Twenty-four hours later, no news outlets had made a fuss about the video, and the Clinton team privately declared victory. + The video clip may have been trivial, but the brief episode surrounding it illustrated how visual and audio technologies like video streaming have the potential to drive political news in unexpected directions, and how White House candidates are aggressively monitoring and trying to master them. + Ambivalence among candidates about communications technology is nothing new; Howard Dean's 2004 presidential campaign used the Internet as a fund-raising A.T.M., then stumbled after the widely circulated video of the so-called Dean Scream. But in the 2008 race, the blessings and curses are much closer to the heart of the candidates' strategies and their determination to control how they are presented to voters. + Mrs. Clinton's campaign, for instance, has already shown that it is determined to use every new media tool to advance her carefully developed image as a centrist, and to re-introduce her to Americans as warmer, more relaxed and confident. + A Republican candidate, former Gov. Mitt Romney of Massachusetts, most recently moved to regain control of his image as a social conservative after being confronted with video clips from 1994 that showed him defending abortion rights and gay rights. As that video moved through the YouTube universe, Mr. Romney responded quickly, saying he was wrong on some issues in 1994, a statement that was swiftly captured on YouTube as well. + ''In previous campaigns you'd think or hope that the threat would just go away, but now it's imperative that you attack back quickly and personally, and with advanced technology,'' said Kevin Madden, an adviser to Mr. Romney. + As with Mrs. Clinton's failure to carry a tune, though, not everyone sees an invariable need for a quick rejoinder. A potential embarrassment popped up recently for former Senator John Edwards of North Carolina when old video surfaced on YouTube showing him fussing with his hair for two minutes. The clip recalled the Breck Girl sobriquet that the White House applied to him in the last presidential campaign. + ''We tossed around some funny ideas to deal with it -- sort of asking, Is this really what matters to the country? -- but we decided it was just not that hot an attack at this point,'' said Mathew Gross, the Internet strategist for the Edwards campaign. ''The question is, How much do campaigns allow themselves to be distracted by this stuff?'' + Several candidates began their campaigns this winter with kickoffs geared entirely toward new media. In late December, Mr. Edwards appeared from New Orleans in a campaign video from YouTube announcing his presidential candidacy, and also appeared that week on Webcasts from Iowa and New Hampshire. Soon afterward, Senator Barack Obama of Illinois and Mrs. Clinton announced their own bids for the White House in taped video messages on their Web sites. + Mr. Obama appeared poised and energetic in his video, choosing an open-collared shirt to convey a youthful spirit (though it was deemed too informal by some political analysts and fashionistas). Mrs. Clinton's video drew even more attention: she looked and sounded more relaxed than usual as she proposed that she and Americans spend 2007 having a ''chat'' about the future of the country. + There are jarring contrasts between the new Clinton video and the old news footage of the headbanded Mrs. Clinton: her physical appearance and her tone of voice, now softer, have changed almost entirely. Yet cable news has been recycling the old clips in recent reports about why Americans might view her as unappealing, or even unelectable. + ''Probably more than anyone else in the field, Hillary Clinton is burdened with certain assumptions that people are making about her politics and her personality,'' said Paul Maslin, Mr. Dean's pollster in the 2004 campaign. ''The upside is that she will have chances to knock down those assumptions and win people over. Part of the challenge is finding the best media formats to do that in.'' + Mrs. Clinton's advisers are moving to replace Hillary 1.0 with the newer model. Two nights after she announced her candidacy, she held live ''Webchats'' on her site, answering questions (which were vetted by her staff) as she sat on a couch and tried to convey a homey touch. Some analysts described the ''conversation'' as an infomercial, and the Republican National Committee dismissed it as ''Hillary's screenplay,'' but her team is promising more. + Mr. Edwards allowed bloggers to attend events on his announcement tour and to interview him, and gave them greater behind-the-scenes access than other campaigns have. + ''You have to be vigilant about the attacks in the new media, but you also have to be vigilant about the opportunities,'' said Mr. Gross, of the Edwards campaign. ''In 2003 and 2004, every other campaign yielded the online and new media advantages to the Dean campaign. That will not happen now.'' \ No newline at end of file diff --git a/nyt-2007_small/1823899.xml.txt b/nyt-2007_small/1823899.xml.txt new file mode 100644 index 0000000..552d6f3 --- /dev/null +++ b/nyt-2007_small/1823899.xml.txt @@ -0,0 +1,32 @@ +McCain Embracing Advisers Whose Ads Once Drew His Ire + +Senator John McCain, intent on succeeding where his freewheeling presidential campaign of 2000 failed, is assembling a team of political bruisers for 2008. And it includes advisers who once sought to skewer him and whose work he has criticized as stepping over the line in the past. + In 2000, Mr. McCain, Republican of Arizona, said the advertisements run against him by George W. Bush, then the governor of Texas, distorted his record. But he has hired three members of the team that made those commercials -- Mark McKinnon, Russell Schriefer and Stuart Stevens -- to work on his presidential campaign. + In 2004, Mr. McCain said the Swift Boat Veterans for Truth advertisement asserting that Senator John Kerry of Massachusetts had not properly earned his medals from the Vietnam War was ''dishonest and dishonorable.'' Nonetheless, he has hired the firm that made the spots, Stevens Reed Curcio & Potholm, which worked on his 2000 campaign, to work for him again this year. + In October, Mr. McCain's top adviser expressed public displeasure with an advertisement against former Representative Harold E. Ford Jr., Democrat of Tennessee, that some saw as having racist overtones for suggesting a flirtation between Mr. Ford, who is black, and a young, bare-shouldered white woman, played by a blond actress. + The Republican committee that sponsored the spot had as its leader Terry Nelson, a former Bush campaign strategist whom Mr. McCain hired as an adviser last spring. In December, just weeks after the Ford controversy broke, Mr. McCain elevated Mr. Nelson to the position of national campaign manager. + Taken together, the moves provide the strongest indication yet that Mr. McCain intends to run a far tougher campaign than the one he ran in the 2000 primary. And they come as he transitions from being a onetime maverick to a candidate seeking to gather his party around him and create an air of inevitability about his prospects for winning nomination. + As Mr. McCain assembles his team, he is also making it that much harder for his Republican challengers by scooping up a significant circle of the party's top talent. + In recent years, Mr. McCain has made a concerted effort to mend fences with Mr. Bush and reassure the Republican base that he is a reliable conservative. But his moves have focused new attention on the extent to which he may risk sacrificing the image he has long cultivated of being his own man, driven by principle rather than partisan politics. + Mr. McCain's advisers said he was not changing. But they were unapologetic about putting together a group dedicated to doing what it takes to reach the White House and employing lessons from his defeat at the hands of Mr. Bush in 2000. + ''This is about winning at the end of the day,'' said John Weaver, Mr. McCain's longtime senior strategist. ''I don't want to be in a knife fight ever again, but if I am, we're going to win it.'' + Mr. McCain's representatives said he would not provide an interview. + Seven years ago, Mr. McCain charmed the news media and the public with his Straight Talk Express bus tour. He had a lean operation befitting an upstart candidacy, and he regularly spoke out against attack advertising, a quaint notion in retrospect. + In the end, he ran his share of confrontational advertisements, once even leveling the ultimate Republican-to-Republican insult: that Mr. Bush was as dishonest as Bill Clinton. But he was perceived as having been knocked back on his heels by the rougher, tougher Bush campaign. + Now Mr. McCain is building a larger organization, bringing together the heart of the bare-knuckled Bush crew once overseen by Karl Rove while keeping most of the advisers who ran his shoestring effort of 2000. + ''It's like an all-star World Wrestling Federation cage match, except that instead of fighting one another, all of the brawlers are on the same team,'' said Steve McMahon, a strategist for the Democratic National Committee. ''There are very few people who play this game at the highest level, and on the Republican side these guys are among the best.'' + Mr. McCain has also hired Brian Jones, an adviser to Mr. Bush's 2004 campaign; Fred Davis, a media consultant for Mr. Bush in 2004; and Steve Schmidt, who oversaw Mr. Bush's 2004 war room, exploiting any tidbit that could help paint Mr. Kerry as a ''flip-flopper.'' + The hires are another signal that the 2008 primary campaign could be a combative one all around. + On the Democratic side, John Edwards, the former senator from North Carolina, has wasted no time attacking Senator Hillary Rodham Clinton's position on Iraq. And Mrs. Clinton's team includes strategists who invented the concept of the modern campaign war room for her husband 15 years ago. But Senator Barack Obama of Illinois drew cheers at a party gathering on Friday when he warned his fellow candidates against attacking one another. + Mitt Romney, a Republican and the former governor of Massachusetts, has hired Alex Castellanos, a onetime Bush strategist who also famously produced the 1990 commercial for Jesse Helms, the former North Carolina senator, in which a pair of white hands crumpled a rejection letter as a narrator said, ''You needed that job and you were the best qualified, but it had to go to a minority because of a racial quota.'' + Given Mr. McCain's history with some of the people on his team, the evolution of his staff may present an early challenge: How does he stay true to the ''Straight Talk'' spirit of his 2000 campaign, which helped him win the stature he has now, while also engaging in the political brinkmanship it can take to win? + The Democratic National Committee is already criticizing Mr. McCain for his hires, issuing a statement this week calling them ''a testament to how far he's gone down the do-anything-to-win path.'' + Tony Fabrizio, a Republican pollster who is not yet allied with a candidate, said Mr. McCain was running the risk of looking ''politically expedient'' and of blunting his brand as ''Senator Straight Talk.'' He said the risk was highlighted by Mr. McCain's recent suggestions that he may not use the campaign finance system he has long championed. + In 2000, Mr. McCain received money from the system, which gives public financing to candidates who agree to strict spending limits. Mr. Weaver, the senior strategist, said Mr. McCain was keeping his options open because others, including Mrs. Clinton, were planning to work around the system. + As Mr. McCain's aides often point out, for all its appeal, the McCain 2000 campaign was a losing one. And they said it would be unfair to suggest that because Mr. McCain was augmenting his team he was somehow preparing to change who he was. + ''There are no negotiations regarding his principles,'' Mr. Weaver said. + In an interview on Friday, Mr. Jones, the campaign communications director, said Mr. McCain was not allowing his distaste over the Swift Boat commercials to interfere with his relationship with Stevens Reed Curcio & Potholm, with whom Mr. McCain has his own decade-long association. In addition, he said, Mr. McCain hired Mr. Nelson because of his breadth of experience in national campaigns. ''The campaign,'' Mr. Jones said, ''is not going to let past contests on the battlefield limit how it's going to go after talent.'' + Presidential politics are rich in fungible allegiances. James A. Baker III ran the primary campaigns of Gerald Ford and the elder George Bush against Ronald Reagan, only to become Mr. Reagan's chief of staff. This year, David Axelrod is serving as a senior strategist for Mr. Obama; he was a senior strategist to Mr. Edwards inhis 2004 campaign. + ''You could dissect any campaign this way: this guy did this ad this one time,'' said Mr. Schriefer, the former Bush media strategist, who will run Mr. McCain's advertising team. ''There's a tremendous history of foes becoming allies.'' + Mr. McKinnon, who led Mr. Bush's advertising group in 2004, said he saw no inconsistency in working for Mr. McCain. Mr. Bush was right for 2000, he said, and Mr. McCain is right for 2008. ''At the end of the day, the campaign will be won or lost on the character of the candidate and his or her core message,'' Mr. McKinnon said. ''Of course, I believe that will be John McCain.'' + Asked if the senator would avoid the attacks he criticized in 2000, Mr. Jones said that while Mr. McCain had yet to declare his candidacy, any campaign he ran would be ''consistent with his beliefs and values.'' \ No newline at end of file diff --git a/nyt-2007_small/1824250.xml.txt b/nyt-2007_small/1824250.xml.txt new file mode 100644 index 0000000..f801ec9 --- /dev/null +++ b/nyt-2007_small/1824250.xml.txt @@ -0,0 +1,8 @@ +Giuliani Steps Closer To a Run in 2008 + +Rudolph W. Giuliani took another step toward running for president by filing a ''statement of candidacy'' with the Federal Election Commission yesterday, declaring that he would seek the Republican nomination if he runs, according to his campaign. + Appearing last night on the Fox News Channel program ''Hannity & Colmes,'' Mr. Giuliani just skirted the line on declaring his candidacy. ''We still have to formally announce it and do a few more things,'' he said, but added that yesterday's paperwork ''is about as close as you're going to get.'' + When asked if he would beat Senator Hillary Rodham Clinton if he won the Republican nomination, he said, ''I'm in this to win.'' Then he added, ''You do this because you believe that you can win the nomination of your party, and then you believe that you're the strongest candidate to win the election for your party.'' + Mr. Giuliani, the former New York City mayor, is also amending a document he already filed with the commission, dropping the phrase ''testing the waters'' from his application, officials on his campaigns said. + Compared with his top rivals in 2008 at this point, Senator John McCain of Arizona and former Gov. Mitt Romney of Massachusetts, Mr. Giuliani has moved more slowly to hire staff members and build a state-by-state political organization. + An aide to Mr. Giuliani said the new filing with the election commission was mostly housekeeping work, bringing Mr. Giuliani's paperwork into line with that of Mr. McCain's and Mr. Romney's. \ No newline at end of file diff --git a/nyt-2007_small/1824408.xml.txt b/nyt-2007_small/1824408.xml.txt new file mode 100644 index 0000000..58e0501 --- /dev/null +++ b/nyt-2007_small/1824408.xml.txt @@ -0,0 +1,22 @@ +White House Bid? Giuliani Is Still Undeclared + + White House Bid? Giuliani Is Still Undeclared + +Former Senator John Edwards stood in the Ninth Ward of New Orleans and declared, ''I'm a candidate for the presidency of the United States.'' Senator Hillary Rodham Clinton proclaimed on her campaign Web site: ''I'm in. And I'm in to win.'' Former Gov. Mitt Romney of Massachusetts plans to formally announce his White House run in Michigan on Feb. 13. + Rudolph W. Giuliani, the former mayor of New York, took two steps closer to a presidential run on Monday, and yet stopped short, once again, of unequivocally throwing himself in the ring. + After filing a candidate statement with the Federal Election Commission, he appeared on Fox News Channel's ''Hannity & Colmes'' for an interview, and Sean Hannity asked, ''You are then officially running to be the next president of the United States?'' Mr. Giuliani's response was cautious: ''We still have to formally announce it and do a few more things. But this is about as close as you're going to get.'' + When asked if he believed he could defeat Mrs. Clinton -- if they received their respective parties' nominations -- Mr. Giuliani said, ''Well, I'm in this to win,'' but he added, ''I have no idea who is going to get the nomination.'' + A few skeptics have wondered whether Mr. Giuliani, who has been crisscrossing the country over the last few months and signaled his interest in the presidency again and again, may ultimately back out -- as he did in 2000, when he contemplated a race for the Senate but withdrew. + Aides to Mr. Giuliani yesterday dismissed the notion that he was equivocating. ''The response across the country has been extremely encouraging, and Rudy has made clear he is moving toward making an official announcement,'' said Katie Levinson, a spokeswoman for Mr. Giuliani's not-yet-official campaign. + Mr. Giuliani established a federal presidential exploratory committee on Nov. 20 to ''test the waters,'' meaning that he could raise and spend money on polls, focus groups and travel but not declare a candidacy or build a substantial war chest. + On Monday, Mr. Giuliani stopped testing the waters when he filed the statement of candidacy. + ''From the F.E.C.'s perspective, he's running for president of the United States,'' said Karl J. Sandstrom, an election lawyer and a former Democratic member of the Federal Election Commission. Typically, he said, a statement of candidacy and a formal announcement of a run occur around the same time. + Bradley A. Smith, an election lawyer and a former Republican member of the commission, said that filing a candidate statement was not a definitive step. ''People in the past have filed that form and then decided not to run,'' he said. ''Filing the form may just mean that he's planning to raise some serious money that would put him outside of what the F.E.C. considers testing the waters.'' + Mr. Giuliani has recently moved to sell Giuliani Capital Advisors, an investment bank that is part of his consulting business, Giuliani Partners. But a candidacy would not necessarily complicate any sale of his assets. + Candidates for president are required to file personal financial disclosure statements with the federal Office of Government Ethics. Those who declare between Jan. 1 and May 15 must file by May 15. Those who declare their candidacies between May 15 and Dec. 31 must file the statement within 30 days of the declaration, according to Vincent J. Salamone, associate general counsel for the ethics office. + The statements require candidates to disclose financial interests, including sources of income, investment holdings and outstanding loans, going back a year from the filing date. Corporate directorships, partnerships and consulting contracts must be disclosed for the previous two years. + Fred Siegel, a historian at Cooper Union who has written admiringly about Mr. Giuliani, said he thought the former mayor was simply being deliberate. ''He moves at his own pace,'' Mr. Siegel said. ''He thinks these things through very carefully, and I think he's playing this out for tactical advantage.'' + Katon Dawson, chairman of the South Carolina Republican Party, said it was ''a lot easier to get in the race than to get out of the race,'' and called Mr. Giuliani's actions so far ''a natural progression of how you would enter a race for president.'' + Mr. Dawson, who spoke warmly of Mr. Giuliani but has not endorsed any candidate, speculated that the former mayor might simply be choosing the right time and place to make an announcement. ''This is not unusual,'' he said. ''Whether it will be at ground zero, or in California or New Hampshire or Iowa or South Carolina, he will announce on his own terms, and pick his own turf.'' + In his Fox interview, Mr. Giuliani affirmed his support for abortion rights but said he would ''appoint judges that interpreted the Constitution rather than invented it.'' He said he approved of domestic partnership laws, but not same-sex marriage. + He said repeatedly, ''We've got to get beyond Iraq,'' and called for more attention to Iran, Syria, Pakistan and Afghanistan. He called for tightening the country's borders and requiring prospective citizens to learn English, but also said that deporting illegal immigrants in large numbers was impractical. \ No newline at end of file diff --git a/nyt-2007_small/1825042.xml.txt b/nyt-2007_small/1825042.xml.txt new file mode 100644 index 0000000..48d0689 --- /dev/null +++ b/nyt-2007_small/1825042.xml.txt @@ -0,0 +1,24 @@ +Today's Version of 'I Will Run' Is Way More Than 3 Little Words + +It can be hard to keep track of everyone running for president, let alone those who are merely ''exploring'' it, ''unofficially running,'' ''testing the waters'' or ''starting a conversation with the American people.'' + This is the stutter-step season of the 2008 presidential campaign. + Candidates keep announcing that they are running or almost certainly running, then, a few days or weeks later, saying so again while the news media dutifully record each step. + This week, for instance, former Gov. Mitt Romney of Massachusetts -- a Republican who last month announced formation of an ''exploratory committee'' -- made more headlines by saying he would announce that he was going to run for president. Next week, in Michigan. And former Mayor Rudolph W. Giuliani of New York declared this week that ''I'm in this to win'' after filing something called a ''statement of candidacy.'' + On Saturday, Senator Barack Obama, the Illinois Democrat who last month announced the creation of his presidential exploratory committee (after telling Tim Russert of NBC in October that he was thinking about maybe running) -- will finally make an ''announcement'' about his intentions at the Old State Capitol in Springfield, Ill. + Why not just state the obvious? + ''It is incumbent on us to keep the announcement top secret,'' said Robert Gibbs, an Obama spokesman. + Mr. Gibbs cited the need to build ''suspense'' in order to ''draw as many reporters as possible into the frigid teen temperatures of Illinois in February.'' + Hint: Mr. Obama and a throng of reporters will travel from Springfield to Iowa and New Hampshire. + There was actually a simpler, less media-saturated time when prospective candidates would just step up onto a podium somewhere and say they were running for president. And then they were running for president, no dilly-dallying, right down to business. + But the process became more convoluted in 1974 with the advent of campaign finance laws that would eventually spur the creation of presidential exploratory committees. The law mandated that anyone spending more than $5,000 while considering a campaign had to declare it with the Federal Election Commission. + ''The process became a vehicle for a candidate to get attention,'' said William Mayer, an expert on the presidential nominating process and an associate professor at Northeastern University in Boston. + Mr. Mayer said the incremental announcement method took hold in the 1988 campaign for the Democratic nomination. He cites the rollouts of Senator Paul Simon of Illinois (who announced on April 9, 1987, that he would announce that he was going to run for president on May 18) and the Rev. Jesse Jackson (a Sept. 7 alert for an Oct. 10 official start). + ''If you're an unknown candidate, your announcement is pretty much the one surefire way to get attention,'' Mr. Mayer said. ''So you might as well try to make it a twofer.'' + Or threefer, or whatever the media will bear, which is a lot, given the swaths of airspace and cyberspace to fill. The practice of politicians publicly belaboring their decisions has only become worse in recent years. + ''Candidates stretch out the announcements because they can,'' said Elizabeth Wilner, the political director for NBC. + Twenty-four-hour cable and Internet saturation, Ms. Wilner said, ''means there will always be someone waiting to report on every infinitesimal word change relating to their candidate status.'' + There are, of course, instances of would-be candidates forming exploratory committees that yield fruitless exploration. Senator Evan Bayh, Democrat of Indiana, formed one in December, only to announce a few weeks later that he was not running. + There are also increasingly rare occasions when candidates dispense with all flirtation. The normally stem-winding Senator Joseph R. Biden Jr., Democrat of Delaware, proved a paragon of efficiency last month when he declared, ''No exploratory committee, I'm running.'' + Of course, Mr. Biden then proceeded to ruin his grand opening by remarking about Mr. Obama's being ''clean'' (which some viewed as racially offensive) and spending the aftermath apologizing for it. + To ordinary Americans, the string-along announcements can be disorienting. There is, after all, something off-kilter about a system in which Senator Hillary Rodham Clinton, Democrat of New York, can say things like ''I'm in, and I'm in to win,'' which sounds like a clear declaration of her intentions and then goes on to establish ''an exploratory committee,'' a much less certain signal. + Rest assured, barring some dramatic development, Mrs. Clinton is running for president. And she will say so formally in some production at a later time, according to her campaign. In the meantime, she will continue her ''conversation'' with the American people in New Hampshire this weekend. \ No newline at end of file diff --git a/nyt-2007_small/1825152.xml.txt b/nyt-2007_small/1825152.xml.txt new file mode 100644 index 0000000..b55be96 --- /dev/null +++ b/nyt-2007_small/1825152.xml.txt @@ -0,0 +1,40 @@ +GIULIANI SHIFTS ABORTION SPEECH GENTLY TO RIGHT + + Giuliani Shifts Abortion Speech Gently to Right + +As he prepares for a possible run for president -- a road that goes deep into the heart of conservative America -- Rudolph W. Giuliani takes with him a belief in abortion rights that many think could derail his bid to capture the Republican nomination. + But in recent weeks, as he has courted voters in South Carolina and talked to conservative media outlets, Mr. Giuliani has highlighted a different element of his thinking on the abortion debate. He has talked about how he would appoint ''strict constructionist'' judges to the Supreme Court -- what abortion rights advocates say is code among conservatives for those who seek to overturn or limit Roe v. Wade, the 1973 court ruling declaring a constitutional right to abortion. + The effect has been to distance himself from a position favoring abortion rights that he espoused when he ran for mayor of New York City, where most voters favor abortion rights. + ''I hate it,'' he said of abortion in a recent interview with Sean Hannity of Fox News. ''I think abortion is something that, as a personal matter, I would advise somebody against. However, I believe in a woman's right to choose. I think you have to ultimately not put a woman in jail for that.'' + For Mr. Giuliani, a Brooklyn-born Roman Catholic who once considered entering the priesthood, the issue has been a source of discomfort throughout his political career, especially during his first bid for mayor of New York nearly two decades ago. + Now, as he courts voters in more conservative areas, Mr. Giuliani is turning to the same nuanced approach he used back then to explain how he can be both for abortion rights, while being morally opposed to abortion. + While Mr. Giuliani also faces obstacles for his stands favoring gun control and gay rights, perhaps no social issue resonates as deeply in the hearts of Christian conservatives as abortion. + In his recent travels, he has directed questions on the issue toward a discussion about judges, saying he would appoint jurists who believe in interpreting, not making, the law: judges, he said, like Chief Justice John G. Roberts Jr. and Justices Antonin Scalia and Samuel A. Alito Jr., who he has said he believed would place limits on Roe v. Wade. + ''On the federal judiciary I would want judges who are strict constructionists because I am,'' he said last week in South Carolina. ''I have a very, very strong view that for this country to work, for our freedoms to be protected, judges have to interpret, not invent, the Constitution. + ''Otherwise you end up, when judges invent the Constitution, with your liberties being hurt. Because legislatures get to make those decisions and the Legislature in South Carolina might make that decision one way and the Legislature in California a different one.'' + On the issue of a disputed abortion procedure called ''partial-birth abortion'' by opponents, he told Mr. Hannity that a ban signed into law by President Bush in 2003, which the Supreme Court is reviewing, should be upheld. And on the issue of parental notification -- whether to require minors to obtain permission from either a parent or a judge before an abortion -- he said, ''I think you have to have a judicial bypass,'' meaning a provision that would allow a minor to seek court permission from a judge in lieu of a parent. + ''If you do, you can have parental notification,'' he said. + Both appear to be shifts away from statements he made while he was mayor and during his brief campaign for United States senator in 2000. Asked by Tim Russert on ''Meet the Press'' in 2000 if he supported President Bill Clinton's veto of a law that would have banned the disputed abortion procedure, Mr. Giuliani said, ''I would vote to preserve the option for women.'' He added, ''I think the better thing for America to do is to leave that choice to the woman, because it affects her probably more than anyone else.'' + And on a 1997 candidate questionnaire from the National Abortion and Reproductive Rights Action League of New York, which Mr. Giuliani completed and signed, he marked ''yes'' to the question: Would you oppose legislation ''requiring a minor to obtain permission from a parent or from a court before obtaining an abortion.'' + Mr. Giuliani's campaign aides say his positions on abortion have not changed, and that his stand on what critics call partial-birth abortions has been mischaracterized, saying he opposed a ban only if it failed to include an exception to protect the life of the mother. But the ban vetoed by President Clinton did include such an exception. + Those who have followed Mr. Giuliani's career say he is unlikely to undergo a radical shift in his views in the manner of Mitt Romney, a Republican rival and former Massachusetts governor who advocated abortion rights until about two years ago. + Fred Siegel, author of ''Prince of the City: Giuliani, New York and the Genius of American Life,'' said Mr. Giuliani would likely be careful to avoid anything perceived as a flip-flop on the issue. + ''Part of his appeal is that he doesn't bend in the wind,'' he said. + But Richard Land, head of the public policy arm of the Southern Baptist Convention, said Mr. Giuliani's position was even more offensive than that of someone who believes abortion is morally acceptable. + ''To say I think it's morally wrong, but I think it's a woman's choice is like saying I'm opposed to segregation but it ought to be left up to the store owner to decide,'' Mr. Land said. ''That's a preference, not a conviction.'' + Rancor coming from both sides of the abortion debate is nothing new to Mr. Giuliani. When he made his first bid for mayor in 1989, he was widely considered anti-abortion. His stance immediately proved problematic in a city heavily in favor of abortion rights. New York was the first state to legalize elective abortions, three years before Roe v. Wade. + Early in the 1989 campaign, he told the city's Conservative Party leaders he was personally opposed to abortion and was for overturning Roe v. Wade, except in cases of rape or incest. At the same time he said he opposed criminal penalties and ultimately saw it as an issue of ''personal morality.'' + The distinctions he drew were subtle to the point of producing conflicting news accounts in which he was alternately described as favoring abortion rights and anti-abortion. + The issue took on more importance that summer after a Supreme Court decision allowed states to put new restrictions on abortion. As criticism mounted, Mr. Giuliani remained silent for weeks after the decision. Finally, in late August of that year, he issued a statement seeking to ''clarify'' his position. + Where he previously asserted that he would stay away from efforts to protect or overturn the state abortion law, he now said he would fight moves to make it illegal. + Critics ascribed his switch to political opportunism. Chief among his critics were his rivals for the mayor's post, the incumbent, Mayor Edward I. Koch, and the eventual winner of that election, David N. Dinkins, both of whom accused him of heeding the advice of his campaign strategists over his conscience. + Mr. Giuliani defended his change, saying it was the result of deep introspection, not political expediency. ''I had to spend time not only thinking about it, but talking about it with my wife and people close to me to focus on it in a closer way than I have in the past,'' he told The New York Times in an interview at the time. + ''This is a guy who is a Catholic, who thought about going into the priesthood, he has close friends who are priests,'' said Mr.Siegel, ''when he realizes he can't get elected in New York if you're anti-abortion and he begins to modify his position, and eventually he goes all the way to being in favor of later-term abortions.'' + Fran Reiter, who was a deputy mayor under Mr. Giuliani and played key roles in two of his campaigns, believes his acceptance of abortion rights was heartfelt, not a political move. After winning the mayor's office in 1993, he supported and strengthened legislation protecting women's access to abortion clinics. + The National Abortion and Reproductive Rights Action League of New York was so convinced, it remained neutral in the 1993 and 1997 mayoral races, and did so again when he briefly ran against Hillary Rodham Clinton for United States in 2000 before dropping out of the race. + ''That tells you how strongly we felt, that we would actually remain neutral when he's running against the first lady of the United States,'' said Kelli Conlin, the state organization's president, who served on Mr. Giuliani's 1993 transition team and on several mayoral commissions. + Now Ms. Conlin and other abortion rights advocates, including Ms. Reiter, say Mr. Giuliani clearly appears to be distancing himself from his former positions. + ''The term strict constructionists has become a transparent code word for nominating judges who would overturn Roe, and both sides know it,'' Ms. Conlin said. ''I think this is a troublesome wink and a nod to the far right of the Republican Party, and that's not the Rudy Giuliani I know.'' + Buddy Witherspoon, Republican National Committeeman from South Carolina, said Mr. Giuliani's recent appearance with party leaders there shifted his and some others' perceptions of the candidate, though not enough to satisfy the most socially conservative among them. + ''All I'd heard from the media is that he was a moderate in support of abortion,'' Mr. Witherspoon said, ''then I heard him say he did not support abortion, however he also said he did not think a woman should go to jail for having had an abortion.'' + Correction: February 28, 2007, Wednesday A front-page article on Feb. 10 about efforts by Rudolph W. Giuliani, a Republican presidential contender, to distance himself from the position favoring abortion rights that he espoused when he ran for mayor of New York attributed an erroneous distinction to New York State. It was the second state to legalize elective abortion -- not the first, which was Hawaii. \ No newline at end of file diff --git a/nyt-2007_small/1825909.xml.txt b/nyt-2007_small/1825909.xml.txt new file mode 100644 index 0000000..195fabb --- /dev/null +++ b/nyt-2007_small/1825909.xml.txt @@ -0,0 +1,8 @@ +Not-So-Friendly Skies + +It is about 1,100 miles from Des Moines to Manchester, N.H., with no direct flights except by private jet. And thanks to new Congressional ethics rules, the trip may feel much longer to some presidential candidates. + Federal election laws allow candidates to use corporate jets and reimburse the owners at the lower cost of a first-class commercial ticket. But recently passed Senate rules would prohibit members, including Senators Hillary Rodham Clinton, Barack Obama and John McCain, from getting the discount. The details will not be clear until the bill returns from a House-Senate conference, and it could ban all federal candidates from getting the discount. But many, including Mr. McCain, Mr. Obama and Mrs. Clinton, have already given it up. + For now, the situation may give an edge to candidates outside the Senate: former Senator John Edwards, above, among Democrats, or former Gov. Mitt Romney of Massachusetts and former Mayor Rudolph W. Giuliani of New York among Republicans. + Mr. Romney's campaign recently sent an appeal to supporters explaining the federal law and asking for the use of private planes. ''One key way in which we can cut costs is by increasing our pool of private planes,'' the campaign's finance director wrote. + DAVID D. KIRKPATRICK + THE CAUCUS: MONEY FOCUS \ No newline at end of file diff --git a/nyt-2007_small/1827213.xml.txt b/nyt-2007_small/1827213.xml.txt new file mode 100644 index 0000000..e0fd0d9 --- /dev/null +++ b/nyt-2007_small/1827213.xml.txt @@ -0,0 +1,26 @@ +No Retreat on Iraq Stance, McCain Insists + +Senator John McCain arrived in Iowa on Saturday to begin testing one of the biggest questions so far of the presidential campaign: Can he win as an all-out supporter of the war in Iraq? + Almost from the moment he took the stage at his first event here since forming his presidential exploratory committee, Mr. McCain made clear that he was not backing down when it came to the war, and that in political terms, he might see an advantage in presenting himself to the first Republican voters in the nominating process as an unapologetic advocate for the war in Iraq. + He attacked Senate Democrats for what he called ''a political stunt'' in trying to join the House in passing a nonbinding resolution opposing President Bush's plan to send more troops to Baghdad. He warned of catastrophic consequences if the United States abandoned its effort in Iraq. + He belittled the White House for falsely raising hopes by asserting early American successes in the war. And he spent nearly 60 minutes fielding questions and challenges from an audience of Republicans, many of them clearly with Mr. McCain on the issue but others anguished over the war. + ''I know how tough it is for the American people, I know how frustrated Americans are, I understand your frustration,'' Mr. McCain said. ''I understand your frustration. But I also want to tell you that I believe if we fail, the consequences of failure are catastrophic.'' + To be here, Mr. McCain skipped the debate in the Senate -- he called the rare Saturday session in the Capitol ''insulting to the public and our soldiers'' -- but the war was as omnipresent in the Des Moines hotel ballroom as it was in Washington. + It brought to life what has become a central strategic gamble for the McCain campaign: that his support of the war in Iraq would help him among the select groups of Iowa Republicans who will caucus here next January. It is a position that gives him an opportunity to reassure Republican activists here who have not forgotten or forgiven his frequent criticisms of the administration in the earlier years of Mr. Bush's presidency, according to state party officials. + ''Even if it's not popular in the general election, everybody in the Republican primary process is supporting the surge,'' said Mark McKinnon, one of Mr. McCain's senior advisers. + In many ways, Mr. McCain finds himself in the opposite situation of Senator Hillary Rodham Clinton of New York. He has embraced a position that may lift him through the primaries, but that could complicate his prospects against an antiwar Democrat in the general election, his own advisers said. + Mr. McCain's aides could barely contain their frustration, as Mr. McCain sailed off on this maiden voyage of his campaign, at the extent to which perhaps the most important factor in determining the 2008 race was beyond of their control. One aide, who was granted anonymity in order to offer a rather gloomy appraisal of Republican hopes for holding the White House in 2008, said that if the situation Iraq deteriorated or even stayed the same, it would be a major liability for the Republican who won the presidential nomination. + ''Republicans are in a tough spot -- there's nobody running for president who controls the destiny of what happens there,'' the aide said. ''I do think as Republicans we have to get to a difference place on Iraq or we are in trouble as a party.'' + Mr. McCain's first stop Saturday suggested just how torn Americans are about this war, regardless of party. A questioner who introduced himself as a veteran of combat in Iraq and Afghanistan drew an ovation that lasted nearly 60 seconds. But moments later, Mr. McCain faced skeptical questioning from Marty Baddeloo, a business coach who lives in West Des Moines and who said he had a son who served in Iraq, about how much money -- and how many lives -- would be needed for Mr. McCain to accomplish his mission. + ''I'd like to know what it's going to cost to be successful in Iraq,'' Mr. Baddeloo said. ''Assuming you are elected president, if your plans are implemented, what is it going to cost in terms of time and dollars for us to be successful in Iraq?'' + Mr. McCain, holding a wireless microphone and wearing a loose-fitting brown leather jacket, answered with a subdued tone. ''I'd like to be able to give you an answer, sir,'' he said. ''And you reflect the frustration that many Americans feel.'' + While the war in Iraq looms as a political liability for the major Republican contenders -- Mitt Romney, the former governor of Massachusetts, and Rudolph W. Giuliani, the former mayor of New York, have also expressed support for Mr. Bush's war policy -- none have been as closely identified with the war effort as Mr. McCain. And Mr. McCain showed no signs of walking away from a war that, his advisers said, he arguably owns as much as Mr. Bush does. + ''I don't know what the other options are,'' Mr. McCain said when one questioner asked what would happen if this escalation of 21,500 troops -- fewer than Mr. McCain had advocated -- failed to work. ''Because if we fail here -- I'm going to give you very straight talk here -- it's not going to maintain the support of the American people. I can give you lots of options, but I'm hard-pressed to find one that will get the support of the American people.'' + And he was disdainful of Senate Democrats, mocking them for pushing through a procedural vote in the Senate on a Saturday. ''I'm more than willing to discuss Iraq and my views, but I think it's insulting to the public and our soldiers to pretend we are discharging our responsibilities in any meaningful way.'' + Mr. McCain said again at a news conference on Saturday morning that political factors played no role in his views. ''I've said all along I cannot be concerned or will not be concerned if it helps me or hurts me with anyone,'' he said. ''Its an issue that transcends any political ambitions I might have.'' + Still, that is something that Mr. McCain's advisers -- thinking about both the primary battle and a general election fight -- have given much thought to. Mr. McCain's advisers said they thought that he could be insulated to some extent by a failure of American policy in Iraq because of the fact that he has long been critical of the way the war was handled; he called for an increase in troop strength there long before Mr. Bush acted. + ''The fact of the matter is McCain has been saying this for three years and maintained from Day 1 that we don't have enough troops there,'' said Dave Roederer, the Iowa chairman of Mr. McCain's campaign. + And Mr. McCain was openly scornful of the White House for raising expectations among the American public by making statements that conveyed that the effort in Iraq was far more successful than it actually was. + ''We raised people's expectations: mission accomplished, a few dead-enders, last throes,'' he said. ''All those comments that made Americans believe we were on the verge of getting this thing done when in reality on the ground, as you saw, it was a long, hard struggle that we should have told people that it is.'' + Mr. McCain's aides said they saw one upside for Mr. McCain now: that his position on the war could help him through the months ahead. + ''In the Republican primary, this is going to be good for Senator McCain,'' said Russ Schriefer, Mr. McCain's media adviser. ''It makes him more of a Republican. To those who might have been uneasy about him not being enough of a team player it shows him as very much a team player on this issue.'' \ No newline at end of file diff --git a/nyt-2007_small/1827929.xml.txt b/nyt-2007_small/1827929.xml.txt new file mode 100644 index 0000000..171c18b --- /dev/null +++ b/nyt-2007_small/1827929.xml.txt @@ -0,0 +1,28 @@ +Time to Throw Their Books Into the Ring + +The lineup of potential presidential candidates is a mishmash of senators, governors, former big-city mayors and a retired four-star Army general. + But nearly all of them share one title: published author. + ''You're not a real candidate, Pinocchio, if you haven't written your own book,'' said Mark Halperin, the political director of ABC News. ''If you know everybody else is doing a book, you've got to do a book.'' + The crowded field of early candidates has created a traffic jam of titles, from the rags-to-riches memoir to the earnest political manifesto. + All of them could be called candidate lit, a publishing minigenre that includes runaway best sellers (''The Audacity of Hope'' by Senator Barack Obama of Illinois) and unqualified duds (''Between Worlds,'' a memoir by Gov. Bill Richardson of New Mexico). + For candidates, writing a book is a way to make money, build gravitas and grab media attention. (They can also use a memoir as a dumping ground for past unpleasantries, paving the way for the campaign-trail line ''I addressed that in my book.'') + For publishers the 2008 campaign season is the time to rerelease forgotten titles, sign unpublished candidates and, if they're lucky, laugh all the way to the bank as they reap sales from best-selling political books. ''What you have, essentially, is a celebrity with built-in press coverage,'' said David Rosenthal, the publisher of Simon & Schuster. Mr. Obama's latest book, for example, has sold more than a million copies in hardcover. + Mr. Obama's publisher, Crown, just reissued his 1995 memoir, ''Dreams of My Father,'' in hardcover to capitalize on renewed interest. In December, Senator Hillary Rodham Clinton's publisher rereleased ''It Takes a Village'' in a 10th-anniversary edition, but it flopped, selling only 6,000 copies, according to Nielsen BookScan, which tracks sales in most bookstores and online retailers. Mr. Richardson's memoir will have another chance to succeed next month, when it is released in paperback. + Mr. Obama's recent success has revived the notion of experiencing the pace and trappings of a presidential campaign through a book tour, said Chuck Todd, editor of the daily political tip sheet Hotline. + ''The book publishing business has become the new exploratory committee,'' Mr. Todd said. ''For Obama, it was a way of testing the waters. That's when you find out: Are you interesting enough to get enough interviews? Can you get people to show up for a signing?'' + The tradition of candidate books dates back at least to the 1950s, when John F. Kennedy, then a United States senator, introduced himself as a viable presidential candidate with ''Profiles in Courage,'' which won a Pulitzer Prize in 1957. + But even now that book is plagued by accusations that it was ghostwritten by Theodore Sorenson, a Kennedy aide. Many politicians employ ghostwriters, and some, like Senator John McCain of Arizona, are careful to give them prominent credit on the covers. + Still, memories of ''Profiles in Courage'' have driven many candidates to produce their own version. ''They all think they're going to capture the 'Profiles in Courage' moment,'' Mr. Todd said. + On the Democratic side Senator Clinton has four books; Senator Obama and former Senator John Edwards each have two; Mr. Richardson and Representative Dennis J. Kucinich of Ohio each have one. Senator Christopher J. Dodd of Connecticut has his first book, ''Letters From Nuremberg,'' coming out in September, an account of the Nuremberg trials, based on letters from his father, who was a prosecutor there. + Senator Joseph R. Biden Jr. of Delaware is writing a book tentatively titled ''A Better Place,'' which a Random House spokeswoman called ''both a memoir and an overview of American politics today.'' (Gov. Tom Vilsack of Iowa has not written a book, but a spokeswoman said, ''He's working on one.'') + The Republicans are similarly wordy. Mr. McCain has four books to his name; former Mayor Rudolph W. Giuliani of New York, former Gov. Mitt Romney of Massachusetts, former Gov. Mike Huckabee of Arkansas, Senator Sam Brownback of Kansas and Representative Tom Tancredo of Colorado have one each. Senator Chuck Hagel of Nebraska just signed a contract with HarperCollins for a ''book about America's future,'' his spokesman said. + There are three basic types of candidate books, said Peter Osnos, the founder and editor at large of PublicAffairs, which has published books by Mr. Obama, former President Bill Clinton and General Clark. + The first is the ''introduction'' book, in which candidates without much name recognition introduce themselves to the public, as in Mr. Edwards's ''Four Trials,'' published as he announced his candidacy for president in 2003. Then there is the ''manifesto'' book, which typically outlines what a candidate would do if elected. The third category includes off-topic books like ''Earth in the Balance'' by Al Gore, and Mr. Dodd's forthcoming Nuremberg account. + Steve Ross, the publisher of Crown, an imprint of Random House, said he was plagued by book proposals that are overly packaged or generic. ''The typical pack of candidate books are group-thought position papers,'' Mr. Ross said. ''Ghostwritten, shallow, totally controlled messages. Most of these books are going to be wastes of trees.'' (They are also open to mockery. Recall Al Franken's candidate-book spoof, ''Why Not Me?,'' written long before he became a real Senate candidate and adorned with a picture of Mr. Franken posing in a plaid shirt next to a golden retriever, and chapters with titles like ''The Courage to Dare'' and ''Daring to Dream.'') + ''If a book is a narrative about who you are or something you believe in, it has to feel authentic,'' Mr. Osnos said. ''It has to feel that you're really writing about yourself. It has to feel honest, to the extent that any public figure is honest.'' + Using a book to predict a candidate's success, or even an undeclared candidate's secret intentions, is a tricky business. Mr. Gore, for instance, has not yet ruled out a presidential run, but his busy publishing activity could be interpreted by well wishers as a positive signal. His publisher rereleased ''Earth in the Balance'' last October, and he has written a young-adult version of ''An Inconvenient Truth'' for release in April. Mr. Gore also has a book coming out in May, ''The Assault on Reason,'' that fits squarely in the manifesto category of candidate lit. + But political insiders caution against reading too much into the tea leaves. While Mr. Clinton and Mr. Gore's 1992 election-year manifesto, ''Putting People First,'' previewed a victory, John Kerry and Mr. Edwards's 2004 book, ''Our Plan for America,'' modeled on the Clinton-Gore book, fizzled with the public. + And though Mr. McCain's 1999 book, ''Faith of My Fathers,'' was a critical success and a best seller, he failed to win the nomination in 2000. ''A Charge to Keep,'' by George W. Bush, was dismissed by critics as an expanded stump speech mostly written by Karen Hughes, his omnipresent communications adviser. + The lesson? ''Candidates can win,'' said Mr. Halperin of ABC News, ''even if their books don't sell well.'' + Correction: February 24, 2007, Saturday An article in The Arts on Thursday about books by presidential candidates misstated part of the title of Senator Barack Obama's 1995 memoir, which has been reissued in hardcover to capitalize on renewed interest. It is ''Dreams From My Father,'' not ''Dreams of My Father.'' The article also misidentified the original publisher of that book and of a book by Bill Clinton. It was Times Books/Random House, not PublicAffairs. + Correction: March 12, 2007, Monday An article on Saturday about fund-raising efforts in New York by Senator Barack Obama misspelled the surname of one of President John F. Kennedy's closest advisers, who introduced Mr. Obama at a fund-raiser. He is Theodore C. Sorensen, not Sorenson. The error also appeared in an article in The Arts on Feb. 22 about books written by candidates, including ''Profiles in Courage,'' which then-Senator Kennedy wrote with guidance from Mr. Sorensen. (The Times has misspelled Mr. Sorensen's surname more than 135 times in headlines and articles during the 50-plus years he has been a Democratic adviser and well-known author.) \ No newline at end of file diff --git a/nyt-2007_small/1829870.xml.txt b/nyt-2007_small/1829870.xml.txt new file mode 100644 index 0000000..284dc0d --- /dev/null +++ b/nyt-2007_small/1829870.xml.txt @@ -0,0 +1,23 @@ +End Our Pain: Hold Primary Right Now + + End Our Pain: Hold Primary Right Now + +Efforts are astir in Albany to have New York join the herd of states swept up in the Feb. 5 stampede. + One after another, clammy with fear of irrelevance, states have abandoned their traditional dates for presidential primaries, and moved up the voting to the first Tuesday in February 2008. New Jersey lawmakers took steps this week to join the crowd. California is headed in the same direction. Now, it seems, so is New York. + The State Senate majority leader, Joseph L. Bruno, says the idea ''makes all kinds of sense.'' A spokesman for the Assembly speaker, Sheldon Silver, said yesterday that his boss would ''seriously consider it.'' The third component of the Albany troika, Gov. Eliot Spitzer, is undecided, a spokeswoman said yesterday, but few will be surprised if he goes along. + Received wisdom is that by advancing the primary to Feb. 5 from its usual perch in March, New York would give a boost to its own: Senator Hillary Rodham Clinton on the Democratic side and former Mayor Rudolph W. Giuliani on the Republican. + Why that should be so is not clear, particularly in the case of Mr. Giuliani. Available evidence suggests that his popularity increases in direct proportion to his distance from New York, specifically from the city, where memories of the 9/10 Rudy have not been entirely zapped by 9/11. + Whatever. Albany seems poised to act. + Normally, it is not our place to advise politicians. But this is an urgent matter. The suggestion here is that everyone forget about Feb. 5. For the voters' well-being, we feel that the primary should be held right away -- next month, even sooner if possible. New York should encourage other states with New Hampshire envy to join it in one huge national primary, ASAP. + Let's get this thing over with. + Doesn't it already feel as if we have been Hillaryed, Obamaed, Rudyed and McCained forever? And we still have 87 weeks to go until Election Day 2008. How much pain can people be asked to endure? + As matters stand, we have a fairly good idea who the candidates are and what they think on many issues. Enough already. + By having a Mega-Super-Blockbuster Tuesday right now, we can winnow the race to one Republican and one Democrat. Doesn't that beat month upon month of bilge from a dozen or more quarters, including the Sabbath gasbags, to borrow Calvin Trillin's memorable description of the jabbering heads on Sunday morning television? + For one thing, we would not have to put up with all these coy professions of candidacy, like the ''announcement preceding the formal announcement'' that Senator John McCain of Arizona made Wednesday night on David Letterman's talk show. At least Mr. McCain was forthright about a phony process in which ''you drag this out as long as you can.'' + WITH an immediate Mega-Super-Blockbuster primary, we could eliminate most of the firsts that have been dangled before us as if they bear life-altering potential. You know: first black president, first woman, first Italian-American, first Hispanic, first Vietnam War veteran, first Mormon -- for that matter, first Mormon with only one wife and first Roman Catholic who has had three wives. + By having the field narrowed quickly, a bunch of people could return full time to the offices to which voters (dare we remind everyone) elected them. Others could go back to making fistfuls of dollars making the same speeches they would have delivered gratis on the campaign trail. + Best of all, voters would be spared the often painful spectacle of candidates -- how to put it? -- evolving as the months wear on. + To cite a few examples, Mrs. Clinton has evolved on the Iraq resolution that she voted for in 2002. Mr. Giuliani has evolved from absolute solidarity with President Bush on war tactics. Former Gov. Mitt Romney of Massachusetts has evolved on quite a few issues. As for the staunchly conservative Mr. McCain, he has somehow evolved into someone held ideologically suspect by some conservative groups. What does he have to do -- sponsor a bill declaring Torquemada's birthday a national holiday? + The Mega-etc. primary would efficiently unburden us of candidates who probably didn't stand a chance in the first place. + And then, free of distractions, we can concentrate on what Americans seem to really care about, like where is Anna Nicole's body, what's the latest on Paris and is Britney in or out of rehab today. + NYC E-mail: haberman@nytimes.com \ No newline at end of file diff --git a/nyt-2007_small/1829955.xml.txt b/nyt-2007_small/1829955.xml.txt new file mode 100644 index 0000000..974d338 --- /dev/null +++ b/nyt-2007_small/1829955.xml.txt @@ -0,0 +1,37 @@ +Most Support U.S. Guarantee Of Health Care + +A majority of Americans say the federal government should guarantee health insurance to every American, especially children, and are willing to pay higher taxes to do it, according to the latest New York Times/CBS News poll. + While the war in Iraq remains the overarching issue in the early stages of the 2008 campaign, access to affordable health care is at the top of the public's domestic agenda, ranked far more important than immigration, cutting taxes or promoting traditional values. + Only 24 percent said they were satisfied with President Bush's handling of the health insurance issue, despite his recent initiatives, and 62 percent said the Democrats were more likely to improve the health care system. + Americans showed a striking willingness in the poll to make tradeoffs to guarantee health insurance for all, including paying as much as $500 more in taxes a year and forgoing future tax cuts. + But the same divisions that doomed the last effort at creating universal health insurance, under the Clinton administration, are still apparent. Americans remain divided, largely along party lines, over whether the government should require everyone to participate in a national health care plan, and over whether the government would do a better job than the private insurance industry in providing coverage. + Looking ahead to the presidential campaign, 36 percent of Americans polled said they had confidence in the ability of Senator Hillary Rodham Clinton, Democrat of New York, to ''make the right decisions on health care,'' while 49 percent said they were uneasy about her. + But Mrs. Clinton retained the confidence of nearly 6 in 10 Democrats on the issue, despite the politically devastating collapse 13 years ago of the national health initiative she helped develop early in her husband's presidency. + The poll helps explain why health care already looms large in the presidential campaign, and in statehouses from California -- where Gov. Arnold Schwarzenegger, a Republican, has proposed a sweeping coverage plan -- to Massachusetts, now instituting a program passed under Mitt Romney, the former governor and current Republican presidential candidate. + John Edwards, the Democratic presidential candidate and former senator from North Carolina, recently unveiled his own attempt at a consensus plan, one that would require everyone to have insurance and require employers to provide it or pay into a fund that would do so. Nearly 4 in 10 said that was a good idea; nearly half said they were unsure. + While Democrats are traditionally strong supporters of expanding health coverage, this survey found many Republicans and independents in agreement. + ''I think everybody should have some kind of health care available to them,'' said Diane Manning, 66, of Vancouver, Wash., who described herself as an independent. + ''I don't necessarily think that socialized medicine is the answer, but I think everyone should have that right,'' said Mrs. Manning, who participated in the poll and agreed to a follow-up interview. ''And there are so many people that don't.'' + Nearly 47 million people in the United States, or more than 15 percent of the population, now go without health insurance, up 6.8 million since 2000. + The poll also found overwhelming support behind the Children's Health Insurance Program, which covers many low- and moderate-income children and is up for renewal in Congress this year. Eighty-four percent of those polled said they supported expanding the current program to cover all uninsured children, now estimated at more than eight million. A similar majority said they thought the lack of health insurance for many children was a ''very serious'' problem for the country. + The nationwide telephone poll of 1,281 adults was conducted Feb. 23- 27, and has a margin of sampling error of plus or minus three percentage points. + The poll found Americans across party lines willing to make some sacrifice to ensure that every American has access to health insurance. Sixty percent, including 62 percent of independents and 46 percent of Republicans, said they would be willing to pay more in taxes. Half said they would be willing to pay as much as $500 a year more. + Nearly 8 in 10 said they thought it was more important to provide universal access to health insurance than to extend the tax cuts of recent years; 18 percent said the tax cuts were more important. + ''I wouldn't want to pay a lot of taxes, but if it was spread out and everyone paid their fair share, it would be fine,'' said Don Galvan, 50, a computer programmer from Ringwood, N.J., who considers himself an independent. ''Everybody should have some kind of medical coverage, in case they or their children get sick. Especially children.'' + Most participants said they were satisfied with the quality of their health care, but there was widespread concern about costs. Nearly half of those with insurance said an employer had cut back on benefits or required them to pay more for their benefits in recent years. + A quarter of those with insurance said someone in their household had gone without a medical test or treatment because insurance would not cover it. Six in 10 of those without insurance said someone in their household had gone without care because of the cost. + More people now see guaranteeing health insurance as important than did so at the end of the Clinton efforts in 1996. At that time, 56 percent polled said it was the government's responsibility to do so, and 38 percent said it was not. In the current poll, 64 percent said the government should guarantee health insurance for all; 27 percent said it should not. + Moreover, an overwhelming majority in the current poll said the health care system needed fundamental change or total reorganization, just as they did in the early 1990s, when a deep recession and soaring health care costs galvanized the public and spurred the Clinton drive. + But now, as then, this concern did not translate into a consensus on what should replace it. + One question offered a choice between the current system and a national health insurance program covering everyone, administered by the government and financed by taxpayers. Thirty-eight percent said they preferred the current system, 47 percent the government-run approach. + Robert Blendon, an expert at Harvard on public opinion and health, said politicians had to find some compromise between these philosophical divisions on the role of government, which are deep-seated in American culture, or ''we're going to have the same train wreck we did before.'' + The Clinton plan, itself an attempt at a compromise, collapsed under attacks from an array of interests, including the insurance industry, which warned that the plan amounted to a big government takeover. + Mr. Blendon noted that many politicians were seeking a blend between the private market and the government in their health plans. + How the Poll Was Conducted + The latest New York Times/CBS News poll is based on telephone interviews conducted Feb. 23 through Feb. 27 with 1,281 adults throughout the United States. + The sample of telephone exchanges called was randomly selected by a computer from a complete list of more than 42,000 active residential exchanges across the country. The exchanges were chosen so as to ensure that each region of the country was represented in proportion to its population. + Within each exchange, random digits were added to form a complete telephone number, thus permitting access to listed and unlisted numbers alike. Within each household, one adult was designated by a random procedure to be the respondent for the survey. + The resultshave been weighted to take account of household size and number of telephone lines into the residence and to adjust for variation in the sample relating to geographic region, sex, race, marital status, age and education. + In theory, in 19 cases out of 20, overall results based on such samples will differ by no more than three percentage points in either direction from what would have been obtained by seeking out all American adults. For smaller subgroups, the margin of sampling error is larger. Shifts in results between polls over time also have a larger sampling error. + In addition to sampling error, the practical difficulties of conducting any survey of public opinion may introduce other sources of error into the poll. Variation in the wording and order of questions, for example, may lead to somewhat different results. + Dr. Michael R. Kagay of Princeton, N.J., assisted The Times in its polling analysis. Complete questions and results are available at nytimes.com/polls. \ No newline at end of file diff --git a/nyt-2007_small/1831480.xml.txt b/nyt-2007_small/1831480.xml.txt new file mode 100644 index 0000000..4cedfc4 --- /dev/null +++ b/nyt-2007_small/1831480.xml.txt @@ -0,0 +1,29 @@ +New York Label May Not Fit All In Giuliani Run + +When Republicans say they are skeptical that Rudolph W. Giuliani, the former mayor of New York, can survive their party's presidential nominating process, they usually point to his record of support for abortion rights, gay rights and gun control. + But there may be a less obvious hurdle that Mr. Giuliani has to overcome: Whether he is too much of a New Yorker for the rest of the country. + Americans like New York City, as officials in both parties are quick to say. Most find it vibrant, entertaining and an object of sympathy and pride since the terrorist attacks five and a half years ago that made Mr. Giuliani the national contender he is today. + But the city, with all its tumult and rough edges, is not for everyone. And few people embody all the complicated facets of New York City as much as Mr. Giuliani. + He is swaggering, brash and opinionated and loves to stick his thumb in the eye of conventional political norms. Those traits won him some acclaim in New York, not to mention a lot of tabloid headlines. But he can also be temperamental, controlling, capricious, volatile and, in the words of Edward I. Koch, a former Democratic mayor who supported Mr. Giuliani in his successful bid for a second term, ''mean-spirited.'' + New Yorkers who elected him twice as their mayor tend to view what others might call rudeness as an endearing trait. But beyond conservative unease with his positions on social issues, two of the biggest questions about Mr. Giuliani's presidential campaign are whether the same qualities that made him iconic in New York will play as well in the rest of the country, and whether voters elsewhere will exhibit the same tolerance as New Yorkers did toward the more colorful aspects of his character and style. + ''Being from New York is a very challenging biographical detail for Giuliani,'' said Nelson Warfield, a conservative Republican consultant who worked for Ronald Lauder, Mr. Giuliani's opponent for the Republican mayoral nomination in 1989. ''In Rudy's story, there is chapter after chapter of politics that is accepted as normal in New York, but that is very strange in the rest of the world.'' + Mr. Giuliani's supporters reject such arguments, saying that his role in leading the city after the attacks of Sept. 11 has redrawn his political biography, and that his travel across the country has bleached some of New York City out of his bones. More than that, they say, he has become older and mellower in the aftermath of Sept. 11. + In many national polls, Mr. Giuliani is now running ahead of Senator John McCain of Arizona, former Gov. Mitt Romney of Massachusetts and the rest of the Republican field, suggesting that voters at this early stage think positively of him even if they know relatively little about him. + ''I've seen just the opposite -- as Rudy goes and sits in the middle of living rooms and pulls up a barstool and talks to people in the country, most people are fascinated to hear about his success as mayor,''' said Anthony Carbonetti, a senior adviser to Mr. Giuliani. + Mr. Koch said what he called the ''bad Giuliani'' -- the one who insulted rivals, refused to meet with critics, was ruthless in attacking political enemies -- could not survive the rigors of a primary contest. But he said Mr. Giuliani was disciplined enough that those voters would judge only what he described as the ''good Giuliani,'' the one who has been on display to the public since Sept. 11. + ''The rest of the country is not going to see Rudy Giuliani at his worst,'' Mr. Koch said. ''Because he's a very smart guy and he can be extremely personable.'' + Still, there is a reason no mayor of New York has been elected president, and it has been nearly 100 years since a born-and-bred New Yorker, Theodore Roosevelt, made it to the White House. The last New York mayor to run for president was John V. Lindsay in 1972, and to say that campaign was short-lived would be to overstate its duration. Mario M. Cuomo, a son of Queens and a former Democratic governor, gave much thought to running for president, but decided not to; there are more than a few similarities between the style and temperament of Mr. Cuomo and Mr. Giuliani. + And while Hillary Rodham Clinton might be a New York senator, she is a transplanted Illinoisan by way of Arkansas and Washington, D.C., who has never lived in New York City and could hardly be more different temperamentally from Mr. Giuliani. + The Giuliani tenure at City Hall was, until the attacks of Sept. 11, a blur of a combative mayor fighting with political opponents, civil rights leaders, voters and reporters. (There is tape; voters will hear and see it.) Those who disagreed with him could encounter the kind of scorn one might expect from an irascible New York cabbie. ''There is something deranged about you,'' Mr. Giuliani said in a three-minute lecture to a New Yorker who called in to his weekly radio show to complain about the city's ban on ferrets as pets. + Some New Yorkers may have winced at the images of Mr. Giuliani shouting a barnyard epithet at a union rally -- or furiously shouting ''quiet!'' to booing supporters at a rally where he was conceding defeat to David N. Dinkins -- but most just accepted it as part of what puts the ''New York'' in the ''the mayor of New York.'' + For New Yorkers, the episode in which Mr. Giuliani announced on television that he was leaving his wife (and his wife responded by insinuating at her own news conference that Mr. Giuliani had been having an affair with his press secretary) was the latest chapter in a rich history of mayors prone to private and public excess. + So was his post-breakup move into the apartment of a gay couple on the Upper East Side. That inspired an article by The Times of London which recounted how the mayor left for City Hall every morning after giving his two hosts a goodbye peck on the cheek -- ''a little kiss, it's cute,'' Howard Koeppel, one of his hosts, told the newspaper -- and how Mr. Giuliani affectionately called Mr. Koeppel ''mother.'' + (How much more New York can you get?) + Mr. Giuliani may be a Republican, but a New York Republican is an entirely different breed than, say, one from Arizona. In New York, one can run at once as a Republican Party candidate and a Liberal Party candidate at the same time, which is what Mr. Giuliani did to win election in a Democratic town. He counted among his closest advisers the state's longtime Liberal Party chairman: a chain-smoking, self-acclaimed political boss named Raymond Harding, renowned in New York City political circles for his ability to land all kinds of patronage. (Two of Mr. Harding's sons were on Mr. Giuliani's payroll.) + And the programs that operated under the huge reach of the government he once ran? ''Republican primary voters are going to learn about the Harvey Milk High School for gay students,'' Mr. Warfield said. + He continued: ''They are going to learn what the Liberal Party is. I think Giuliani's endorsement of Cuomo will be part of the primary conversation.'' + Merle Black, a political science professor at Emory University in Atlanta, said he wondered if Mr. Giuliani could maintain this even keel through a campaign that promises to be so long and so competitive. + ''That's the test,'' Mr. Black said. ''He's got to go through two years without reverting to something which might be O.K. in private but it wouldn't be O.K. in Atlanta.'' + POLITICAL MEMO + Correction: March 12, 2007, Monday A Political Memo article on Thursday about the challenges Rudolph W. Giuliani faces in running for president as a former New York City mayor referred incorrectly to the presidency of Theodore Roosevelt, also a New York City mayor. He served more than 100 years ago, from 1901 to 1909; his presidency was not ''almost 100 years ago.'' And although he was indeed the last president born in New York City, he was not the last ''born-and-bred New Yorker'' to make it to the White House. His distant relative, President Franklin D. Roosevelt, was born and raised in Hyde Park, N.Y. + Correction: March 13, 2007, Tuesday A Political Memo article on Thursday, about the challenges Rudolph W. Giuliani faces in running for president as a former New York City mayor, referred incorrectly to the timing of the two-term presidency of another New Yorker, Theodore Roosevelt. He began serving more than 100 years ago, in 1901; his presidency was not ''almost 100 years ago.'' A correction in this space yesterday referred incorrectly to Mr. Roosevelt's time as a public official in New York City. He was a police commissioner, not a former mayor. \ No newline at end of file diff --git a/nyt-2007_small/1832543.xml.txt b/nyt-2007_small/1832543.xml.txt new file mode 100644 index 0000000..14688ef --- /dev/null +++ b/nyt-2007_small/1832543.xml.txt @@ -0,0 +1,31 @@ +Early Primary Rush Upends '08 Campaign Plans + + Early Primary Rush Upends ’08 Campaign Plans + +The trickle of states moving their 2008 presidential primaries to Feb. 5 has turned into an avalanche, forcing all the presidential campaigns to reconsider every aspect of their nominating strategy -- where to compete, how to spend money, when to start television advertising -- as they gird for the prospect of a 20-state national primary day. + In the last two weeks, Senator John McCain, Republican of Arizona, dispatched the director of his political action committee to run his primary campaign in California, where a bill to move the primary to Feb. 5 is on the desk of Gov. Arnold Schwarzenegger. John Edwards, the North Carolina Democrat, announced that he had won the endorsement of Richard J. Codey, a former acting governor of New Jersey, testimony to the state's new status as it readies to shift its primary to Feb. 5 from June. + Senator Barack Obama, Democrat of Illinois, held a rally the other day in Texas, and aides to Rudolph W. Giuliani, the New York Republican, said staff members would be sent to California, Florida and Missouri, as both candidates prepare for expected Feb. 5 primaries in those states. + ''It's becoming a brush fire out there,'' Mr. Obama said in an interview. + Mr. McCain, remarking on the difference from the last time he ran for president, suggested that the front-loaded primary day was not a good development. ''I don't think there's enough exposure of the candidates the way that there used to be, having to go state by state by state over a long period of time,'' he said. + For the most part, the candidates and their aides cannot quite figure out what all this turmoil means for them. The changes, which are shaping up to be the most substantial alteration ever to a campaign calendar in a single election cycle, have heightened the volatility of the most wide-open presidential race in 50 years, one with large and well-financed fields of contenders. + Aides to the candidates said they were debating whether the changes would mean that the nominations would effectively be settled on Feb. 5, by which point easily 50 percent of the delegates are likely to have been chosen, or whether a few strong candidates would divide the Feb. 5 take, forcing the campaign to stretch on for months. That could, oddly enough, make those fewer states sticking to later primaries vital players in the election cycle. + The changes are forcing candidates to decide whether Iowa and New Hampshire, two states with contests before Feb. 5, will become more influential as contenders look for early victories to give them momentum. And with as many as 23 states voting on a single day -- more states than are typically considered competitive in a general election -- candidates must decide which ones to ignore, given the demands on their time and bank accounts. + ''This primary season is turning into the most challenging Rubik's Cube that we've faced in our lifetime,'' said Benjamin L. Ginsberg, the counsel to Mitt Romney, the former governor of Massachusetts who is seeking the Republican nomination, and one of his party's leading experts on election law, pointing to the calendar, the fund-raising demands and the absence of a front-runner in either party. + For Democrats, the prospect of a mega-primary has created a new calculation about the importance of black voters, already a constituency being fiercely courted by Mr. Obama, who is seeking to become the nation's first black president, and Senator Hillary Rodham Clinton of New York. There are hardly any black voters in Iowa and New Hampshire; by contrast, they could play an important role in California, Illinois, Missouri, New Jersey and New York. + The early primary drive is the latest evidence of the national parties' continuing decline in influence over the nominating process. A Democratic Party effort to force states and candidates to abide by the calendar, with threats of refusing to seat delegates chosen by states that defy its rules, seems doomed to fail, with candidates and states saying they will ignore it. + The developments have stirred despair among some Democratic National Committee officials, who pushed through a new calendar this year that sought to dilute the influence of Iowa and New Hampshire by letting Nevada and South Carolina hold nominating contests before Feb. 5. Nevada's Democratic caucus will be five days after the Jan. 14 Iowa caucus, and South Carolina's primary will be at least a week after the Jan. 22 New Hampshire primary. + Donna Brazile, one of the Democratic Party leaders involved in pushing through those changes, said she believed that Iowa and New Hampshire were now more powerful than ever because of the move toward Feb. 5. ''I am very alarmed,'' Ms. Brazile said. ''This is the opposite of what we are trying to do.'' + But Tom McMahon, the executive director of the Democratic Party, said in an interview that having Nevada and South Carolina go earlier had allowed the party to achieve its main goal. ''We've been able to insert diversity where diversity didn't occur before,'' Mr. McMahon said, ''and we are able to preserve more small states to allow more candidates to get into this.'' + With 11 months before the Iowa caucuses, what is most striking, campaign officials said, is just how much uncertainty there is about this most fundamental part of a campaign: when people are going to vote. The National Association of Secretaries of State reported that 23 states were either considering moving to Feb. 5 or certain to do so. But that number changes daily as bills move between legislative committees and to governors' desks. + The importance of Iowa and New Hampshire has emerged as one of the critical questions for the campaigns. + Some analysts said the winners in the early states would emerge with so much momentum and favorable news media attention that they would dominate the national primary to follow and lay claim to their party's nomination, much the way Senator John Kerry of Massachusetts captured the Democratic nomination in 2004 after his victories in Iowa and New Hampshire. + But others said having a 20-state primary a week later would allow candidates who performed weakly in the early states to rebound, particularly if they had the advantage of money or name recognition. + Mr. Giuliani's aides suggested they might not spend as much time and money in Iowa and New Hampshire as other candidates, given his potential strength on Feb. 5 in places like California and New Jersey, two states with a more moderate electorate than Iowa and South Carolina. + ''We have the ability to play the game a little differently,'' said Mike DuHaime, who is running Mr. Giuliani's campaign. ''It's not a matter of saying the early states aren't important, because they are. It is just a matter of realizing that, unlike past primaries, there are many more states this year that will help decide the nominee.'' + Terry Nelson, Mr. McCain's campaign manager, said Mr. Giuliani's campaign appeared to have adopted what he called a Feb. 5 strategy, which he said could be a dangerous miscalculation by ignoring the states deciding before then. ''It doesn't diminish the influence and impact of the earlier states,'' Mr. Nelson said of the shift to Feb. 5, ''because it's going to be very difficult for any campaign to build the resources you need to compete in all of these states.'' + On the Democratic side, aides to Mr. Edwards are hoping for big victories in Iowa and South Carolina to make up for any advantage Mrs. Clinton and Mr. Obama have because they are so much better known and may prove to be more effective at raising money. + There is near-universal agreement among officials of both parties that the new calendar will give a huge advantage to well-known candidates, in particular Mrs. Clinton, Mr. Giuliani, Mr. McCain and Mr. Obama. Beyond that, California and New Jersey are likely to be more receptive to Mr. Giuliani than are Iowa and South Carolina, with their many conservative voters. + The uncertainty goes beyond how many states will move their primaries to Feb. 5, and it seems to be starting a war between the states. + ''California wants a piece of the presidential primary action, and it is willing to harm the country to get it,'' The New Hampshire Union Leader said in a blistering editorial attacking California for encroaching on what had been New Hampshire's early-primary turf. + New Hampshire officials are threatening to move their primary to before Jan. 22, asserting that the shift by Nevada violated a New Hampshire law requiring that it be first in the country. Iowa officials have responded by saying that if New Hampshire moves its primary, it will move its caucus to eight days before the primary. + That has set off a reaction in Michigan, the state that started pushing for others to go early in the first place. ''It's a terrible thing, it's a real problem,'' said Mark Brewer, chairman of the Michigan Democratic Party. ''We're going on the 9th unless some state, such as New Hampshire, breaks the scheduling rules, and then we're going to move it up.'' \ No newline at end of file diff --git a/nyt-2007_small/1832750.xml.txt b/nyt-2007_small/1832750.xml.txt new file mode 100644 index 0000000..ceac427 --- /dev/null +++ b/nyt-2007_small/1832750.xml.txt @@ -0,0 +1,40 @@ +G.O.P. Voters Voice Anxieties On Party's Fate + +After years of political dominance, Republican voters now view their party as divided and say they are not satisfied with the choice of candidates seeking the Republican presidential nomination in 2008, according to the latest New York Times/CBS News poll. + In a survey that brought to life the party's anxieties about keeping the White House, Republicans said they were concerned that their party had drifted from the principles of Ronald Reagan, its most popular figure of the past 50 years. + Forty percent of Republicans said they expected Democrats to take control of the White House next year, compared with 46 percent who said they believed a Republican would win. Just 12 percent of Democrats said they thought the opposing party would win the White House. + Even as Republican voters continued to support President Bush and the war in Iraq, including the recent increase in the number of American troops deployed there, they said a candidate who backed Mr. Bush's war policies would be at a decided disadvantage in 2008. And they suggested that they were open to supporting a candidate who broke with the president on a crucial aspect of his Iraq strategy. + Asked what was more important to them in a nominee, a commitment to stay in Iraq until the United States succeeds or flexibility about when to withdraw, 58 percent of self-identified Republican primary voters said flexibility versus 39 percent who said a commitment to stay. The three leading Republican candidates are strong supporters of the war and the increase in American troops there. + The poll, which was designed to survey more Republicans than it normally would to provide a better statistical look at the mood of the party, found signs that members were uneasy about its future. + ''There is going to be so much antiwar in the news media that there is no way the Republicans are going to win,'' Randy Miller, 54, a Republican from Kansas, said in a follow-up interview after participating in the poll. ''The Democrats will win because of the war. I think the Republicans just won't vote.'' + Compared with the Democrats, Republicans appear far less happy with their choice of candidates for 2008 and are still looking for someone who can improve the party's prospects, the poll found. + While nearly 6 in 10 Democratic voters in the poll said they were satisfied with the candidates now in the race for their party's nomination, nearly 6 in 10 Republicans said they wanted more choices. Yet the poll found that a substantial number of Republicans did not know enough about their leading contenders -- Senator John McCain of Arizona; Rudolph W. Giuliani, former mayor of New York; and Mitt Romney, former governor of Massachusetts -- to offer an opinion of them. + ''I think the Republican candidate has not appeared yet,'' said Richard Gerrish, 69, a Republican from Green-acres, Fla. ''The ones we have now will run out of steam. Someone will come along later that will do better.'' + For all that, the poll found that Republican voters remain largely loyal to Mr. Bush and his positions on the issues. Among Republicans, 75 percent approve of his job performance, and by overwhelming numbers they approve of his handling of foreign policy, the war in Iraq and the management of the economy. + Propelled by this Republican support, the poll registered an increase in the percentage of Americans who say they approve of Mr. Bush's performance; it has increased to 34 percent now from 29 percent last month. + The poll highlights a Republican weakness going into next year's election. Just 34 percent of all respondents said they had a favorable view of the Republican Party, and that is the lowest it has been since December 1998. By contrast, 47 percent of respondents said they had a positive view of the Democratic Party. + And by a 20-point margin, respondents said that if the election were held today they would vote for an unnamed Democrat for president rather than a Republican. Such questions are hardly predictive of the outcome of an election so far away, but they do offer an insight into the health of the party today. + Even as Republicans said they supported Mr. Bush's performance, they showed divisions over the party's ideological makeup; 39 percent of Republican voters said they wanted the next Republican presidential nominee to continue with Mr. Bush's policies; 19 percent said they wanted the next president to become less conservative, and 39 percent more conservative. + ''I think he's spending too much money,'' said Marjorie Bickel, a Republican from Indiana. ''The money's going to have to come from somewhere, and I think they'll raise the taxes and take the money out of Social Security, which they shouldn't.'' + Republican strategists said they were not surprised about the poll's findings, though they said Republicans were too pessimistic in concluding now that the party could not win in 2008. + ''People should be concerned -- we've had a tough last year and a half or so,'' said Glenn Bolger, a Republican strategist. ''But if you go back in time to 1991, the Democrats had a lot of the same concerns, both about the candidates running and their possibility of winning. And it turned out pretty well for them.'' + The national telephone poll was conducted Wednesday through Sunday with 1,362 adults, including 698 Republicans. The margin of sampling error for all adults is plus or minus three percentage points and four percentage points for Republicans. + The poll also found an increase in approval of the way Mr. Bush is managing the war in Iraq, to 28 percent from 23 percent, and how he is handling foreign policy. But at a time when the administration has come under fire for the way veterans from Iraq have been treated at Walter Reed Army Medical Center, 76 percent of Americans, including 57 percent of Republicans, said the Bush White House had not done all it could to deal with the needs and problems facing returning military personnel. + The poll suggested that opinions were still fluid about three of the leading Republican presidential contenders. Mr. Giuliani is the best known of the candidates, but 41 percent of Republicans said they had not formed an opinion of him; 50 percent said they had a favorable rating of him, compared with 9 percent with an unfavorable view of him. + Fifty percent of Republicans said they did not know enough about Mr. McCain to offer a view on him or were undecided, even though he is running for president for a second time. Of the remainder, 32 percent said they had a favorable view of him, compared with 18 percent who said they had an unfavorable view. + By a margin of 43 percent to 34 percent, Republican primary voters said they would prefer to see Mr. Giuliani win the nomination over Mr. McCain, but those kind of measurements taken this early in a campaign tend to be largely discounted by professional pollsters. + Republican primary voters have a definite idea of what they are looking for in a candidate: They want a presidential contender who will make it more difficult for women to obtain abortions, who opposes same-sex marriage and who will push for more tax cuts, the poll found. + The poll found that Republicans think it might be more difficult winning an election as a Mormon, which Mr. Romney is, than as a candidate who had gone through multiple divorces, a category that includes Mr. Giuliani and Newt Gingrich, the former House speaker who is considering a run for president. + Thirty-nine percent of Republican voters thought Americans would not vote for someone with multiple divorces; by contrast, 51 percent of Republican voters thought that Americans would not vote fora Mormon. (Among the general electorate, 42 percent of respondents said Americans would not vote for someone who had been divorced more than once, and 53 percent said most people would not vote for a Mormon.) + Republicans, the poll found, are satisfied, but not enthusiastic, with how Mr. Bush is handing the war in Iraq, taxes and abortion. They said they believed the United States was correct in entering Iraq in the first place, supported the troop escalation pushed by President Bush and believe the war is going well there now. + On an issue that has come to overshadow Senator Hillary Rodham Clinton's campaign to win the Democratic nomination, 52 percent of respondents said Mrs. Clinton had not made a mistake in voting to authorize the war in Iraq, compared with 41 percent who said it was a mistake. + But among Democratic voters, 53 percent said she had made a mistake, underscoring the political problems the war has caused her. Mrs. Clinton has refused calls from Democrats to apologize for that vote; just 16 percent of respondents, and 21 percent of Democratic voters, said Mrs. Clinton should acknowledge publicly that the vote was a mistake. + How the Poll Was Conducted + The latest New York Times/CBS News Poll is based on telephone interviews conducted March 7 through March 11 with 1,362 adults throughout the United States. + The sample of telephone exchanges called was randomly selected by a computer from a complete list of more than 42,000 active residential exchanges across the country. The exchanges were chosen to ensure that each region of the country was represented in proportion to its population. + Within each exchange, random digits were added to form a complete telephone number, thus permitting access to listed and unlisted numbers alike. Within each household, one adult was designated by a random procedure to be the respondent for the survey. + For purposes of analysis, self-identified Republicans were oversampled in this poll. Phone numbers at which a self-identified Republican had been interviewed in polls over the last seven months were recalled and a random member of the household was interviewed. If that person was a Republican, he was included in the oversample. All Republicans were then weighted back to the party's usual proportion of identifiers in the poll, as determined by the average proportion in four polls taken since Jan. 1, 2007. + The results have been weighted to take account of household size and number of telephone lines into the residence and to adjust for variation in the sample relating to geographic region, sex, race, marital status, age and education. + In theory, in 19 cases out of 20, overall results based on such samples will differ by no more than three percentage points in either direction from what would have been obtained by seeking out all American adults. For smaller subgroups, the margin of sampling error is larger. For the 698 Republicans, for instance, it is plus or minus 4 points. + In addition to sampling error, the practical difficulties of conducting any survey of public opinion may introduce other sources of error into the poll. Variation in the wording and order of questions, for example, may lead to somewhat different results. + Dr. Michael R. Kagay of Princeton, N.J., assisted The Times in its polling analysis. Complete questions and results are available at nytimes.com/polls. \ No newline at end of file diff --git a/nyt-2007_small/1833875.xml.txt b/nyt-2007_small/1833875.xml.txt new file mode 100644 index 0000000..6640f28 --- /dev/null +++ b/nyt-2007_small/1833875.xml.txt @@ -0,0 +1,40 @@ +The Future President, On Your Friends List + + The Future President, on Your Friends List + +HAVING already launched a generation of Gwen Stefani clones and death-metal bands into fleeting Internet fame, MySpace -- the largest social-networking site -- is now setting its sights higher: to help elect the next president of the United States. + This week, the site will introduce a section dedicated to politics, with an emphasis on the 2008 presidential election. Called the Impact channel, it will be an online version of a town square, a collection of links to political MySpace pages that will make it easier for the site's 60 million American users per month -- many of them from the traditionally elusive and apathetic youth demographic -- to peruse the personal MySpace pages of, so far, 10 presidential candidates. + The channel will be much like those on the site already devoted to music or video. By clicking into it and on the separate campaign pages, users will be able to read candidate's blogs, view their personal videos and snapshots, and link to other sites that discuss pet issues. Then, theoretically, users will add their favorite candidates to their friends list, and their friends will add them, too. The campaigns will spread virally, in the 2008 campaign strategy of the moment. + Some observers believe that such efforts by MySpace and other social networking sites might make them influential among voters in 2008. Or, in tech language, such sites aspire to be the killer aps of this election cycle, reminiscent of what talk radio (particularly Rush Limbaugh) was in 1994, when it whipped up enthusiasm for the Republican landslide in the midterm elections, or what MoveOn.org was in 2004 when it emerged as a potent force to raise funds and drum up volunteers for the Democratic Party. + ''Part of the motivation is seeing how Howard Dean jumped on the blogging trend in 2003 and established that as a powerful force,'' said Lee Rainie, director of the Pew Internet & American Life Project. ''A lot of candidates don't want to miss the boat on what could be the next big thing.'' + Mr. Rainie added: ''There's now a sense in the political establishment and the consultant community that this year feels qualitatively different in the online world. So rather than being tepid in their embrace of it, they're grasping it in a bear hug.'' + Conversely, while other the social networking sites have welcomed their rising political profiles, the move by MySpace (which comScore Media Metrix reports has nearly four times the number of individual users in the United States as Facebook, its closest rival) is the most notable bid so far to establish a presence in the 2008 race. + From MySpace's perspective, the move is a way to stay competitive within a rapidly evolving environment in which both candidates and rival online companies are making moves to dominate the virtual landscape for the next election. + Not surprisingly, MySpace executives predict that the site, founded in 2003, can help candidates influence younger swing voters. Some 86 percent of its American users are voting age, and even younger users are on campaign coordinators' radar. ''The election is a little less than two years away,'' said Christian Ferry, the national e-campaign director for John McCain. ''A 16- or 17-year-old today is going to be eligible to vote in 2008.'' + Tom Anderson, 31, a MySpace founder, said, ''MySpace has a method of reaching people who are historically not interested in voting'' and may not read newspapers or watch news on television. He added: ''A MySpace profile could excite their interest in ways they are used to. In the same way they learn about their friends, they could learn about a candidate.'' + The Impact channel will also feature voter registration tools that are intended to function as a cyber version of the Rock the Vote youth drives of recent years. And perhaps most enticing to the campaigns, the site will start a one-click payment function to help contenders solicit campaign contributions. This online tool will appear on each candidate's official MySpace page, and can be easily dragged onto a user's own page. And from there it can be dragged to a user's friend's page, theoretically as easily as MySpace members now swap songs by their favorite bands, said Mr. Anderson and his fellow founder, Chris DeWolfe. + In the same fashion, they can drag candidate ads onto their own pages -- ''digital yard signs, for lack of a better term,'' Mr. DeWolfe, 41, added. + An official announcement of the Impact channel will be made on Monday, they said. + Already, campaigns are showing interest in the potential of the site. + ''MySpace is definitely one of the tools we'll be using to engage Internet users,'' said Jen Psaki, a spokeswoman for the Barack Obama campaign, ''and we're well aware that young people are the ones who are engaging the campaign through the Internet, more so than other age brackets.'' + Besides Mr. Obama, the Democratic contenders John Edwards, Joseph R. Biden and Dennis J. Kucinich have set up MySpace pages, as have the Republicans Ron Paul, the Texas congressman, and Mr. McCain. In coming weeks, Hillary Rodham Clinton, Rudolph W. Giuliani and Mitt Romney will also launch their official MySpace pages, a MySpace spokeswoman said. + The candidates were moving onto MySpace even without the lure of the Impact channel, which is expected to be operating Monday. + Several candidates already have pages posted on competitor sites like Facebook, and many, like Mr. Obama, have campaign staffers aggressively trying to co-opt social networking concepts, like viral marketing, through their own independent Web sites. + Mr. Edwards, for example, not only announced his candidacy in December on YouTube, and has been blogging away on his own site, Facebook and MySpace ever since. Lately, the former North Carolina senator has even been stumping via avatars on the virtual reality landscape of Second Life. + In fact, just a few weeks ago, YouTube, which did not exist during the 2004 election cycle, introduced its own campaign-oriented channel on the site, YouChoose '08. + But the cultural position that MySpace occupies promises to create a curious -- and perhaps potent -- collision of ideologies and stylistic sensibilities. While Facebook got its start as a means to link Harvard students, and to this day maintains an exclusive, clubby feel (that's why many loyal users prefer it to MySpace), MySpace is a colorful, hormone-soaked collage. It has all the graphical elegance of a wall full of graffiti, and members tend to communicate in fluent dude-speak. + And now the kids are inviting the adults to the party. + The humorous disconnect has already been noted by Jon Stewart, who last Wednesday ran a satirical segment on ''The Daily Show'' about a hypothetical MySpace page for presidential contender Christopher J. Dodd, the Democratic senator from Connecticut. + That fictional site was decorated in high MySpace fashion, with colorful unicorn-festooned wallpaper and a highly open friends list that happened to include Richard Ramirez, the convicted Night Stalker killer. The site included the classic MySpace incriminating spring-break-style photos, including an altered one of the senator, in a pinstriped suit, sucking on a beer bong while bikini-clad collegians cheered on. + ''Oh boy, that's not going to be good,'' joked Mr. Stewart, regarding the photo. + So far, the real candidate MySpace pages are considerably less vivid. + The unofficial Hillary Clinton site -- created by a supporter in Seattle but lauded by her campaign representatives -- portrays less-than-spontaneous snapshots, like one of the New York senator posing before Niagara Falls in a gray business suit. In classic MySpace fashion, it lists her ''status'' (married), what she is ''here for'' (networking, friends), and her zodiac sign (Scorpio). + Like other social networking sites -- which include Facebook, Friendster, Xanga and Tribe -- membership is free with MySpace. + Anyone, even a future president, can register with the site in minutes, even come up with a cutesy online moniker. The implicit goal for most members is to reach out to as many people as possible and add them to their Friend Space -- which in politics is known as a candidate's base. Friends, who are added by permission only, often do not meet offline, but do readily swap links to favorite songs, videos, jokes and photos. + This is the much-lauded viral nature of the medium that has managers of presidential campaigns intrigued. In fact, it has already started to yield dividends, according to Peter Daou, the Internet director for the Clinton campaign, even weeks before the senator unveils her official MySpace home page, which will include an empowerment theme aimed at girls and young women called ''I Can Be President.'' + Mr. Daou pointed out that Mrs. Clinton already has spread her message to more than 25,000 ''friends'' on her unofficial MySpace page. (Mr. Obama has more than 64,000 on his official site.) + Arguably, the push into presidential politics may buy MySpace a bit more time to be relevant in a landscape where every technology seems moments away from obsolescence. + Already, critics like Michael Hirschorn, in the current issue of The Atlantic Monthly, are pondering what happens when MySpace becomes just another place to find blogging and video upload capability. ''Thanks to the inexorable process of Web innovation,'' Mr. Hirschorn wrote, ''such stuff goes from 'OMG' to 'whatever' in no time flat.'' + The same could be true for the sudden ascendance of social networking in the political process as well, observers said. + ''Right now, social networking and YouTube are the two most striking things about this election cycle that make it different from 2004 -- so far,'' said Mr. Rainie from the Pew Project. + ''That could change,'' he said. ''In 10 months, we could be talking about Second Life or text-messaging as 'the next big thing.' '' \ No newline at end of file diff --git a/nyt-2007_small/1835632.xml.txt b/nyt-2007_small/1835632.xml.txt new file mode 100644 index 0000000..629218b --- /dev/null +++ b/nyt-2007_small/1835632.xml.txt @@ -0,0 +1,35 @@ +Revealing All Before Anyone Else Can + + Revealing All Before Anyone Else Can + +WHEN confronted with the prickly question of whether to disclose their illnesses, Presidents Calvin Coolidge, Woodrow Wilson, Franklin D. Roosevelt and John F. Kennedy divined a simple strategy, historians say. They dissembled. They lied. They covered up or simply kept their mouths shut to keep Americans in the dark. + One can only imagine that those presidents would be rubbing their eyes in disbelief this week. + Not only at the sight of John Edwards, the North Carolina Democrat and presidential candidate, telling the nation in a news conference that his wife's cancer had returned in an incurable form. But also at the slew of other presidential contenders who are dealing with illness in a decidedly 21st-century fashion. + Senator Hillary Rodham Clinton, the New York Democrat, regularly discusses her husband's heart surgery in her speeches on the campaign trail. Mitt Romney, the former governor of Massachusetts, and his wife, Ann, appeared this month on CNN's ''Larry King Live'' to discuss her struggles with multiple sclerosis. + Rudolph W. Giuliani, the former mayor of New York, routinely tells voters about his battle with prostate cancer. And Senator John McCain, the Arizona Republican whose face bears the imprint of his bouts with skin cancer, joked recently that he has ''more scars than Frankenstein.'' + In weighing the age-old question of how to confront serious illness, presidential candidates and their spouses are increasingly opting to come clean. Many are tossing aside traditional notions that have suggested that public airings of such conditions might sink a campaign or derail a presidency. + Some say the shift reflects the greater freedom candidates have in modern times to portray themselves as more human and vulnerable. Others say the public confessions are driven by a desire to control the political message before reporters do. With Internet bloggers, cable news channels and around-the-clock news cycles, keeping such conditions safely buried in the closet is close to impossible, they say. + ''These days, given all of the ways that people get their information, you've got to assume there are no secrets,'' said Michael S. Dukakis, whose wife, Kitty, publicly discussed the spinal surgery she underwent during his 1988 presidential campaign as well as her previous addiction to prescription diet pills. + ''Your life is an open book,'' Mr. Dukakis said. ''You can't hide anything.'' + But political analysts also see something of a tactical calculation here. They say candidates are increasingly humanizing their campaigns to connect more deeply with voters who hunger for more intimate and personal glimpses of their elected leaders. + ''The public has greater sympathy now for someone in politics who is struggling with health concerns,'' said Robert Dallek, the presidential historian. ''People in this country: they go through divorces, they have their psychological and health problems. They want to see that in their candidates, too.'' + John C. Green, a political analyst at the University of Akron, described Mr. Edwards and his wife, Elizabeth, as courageous for revealing that she had metastatic, or Stage 4, breast cancer, which had spread beyond the breast and lymph nodes to the bone. + But Mr. Green said the announcement also seemed carefully orchestrated to have ''a positive impact'' on Mr. Edwards' campaign. He noted that the news conference was held at the Carolina Inn, the Chapel Hill resort where the Edwardses had their wedding reception 30 years ago, a symbolic locale that might reinforce the image of a devoted married couple in the minds of voters. + ''It most certainly made sense for the campaign to address this, but it was also pretty good theatrics,'' said Mr. Green, who heads the Ray C. Bliss Institute of Applied Politics at the University of Akron. ''They did something that was probably necessary but in a positive way for the campaign.'' + In the past, however, presidents and presidential candidates have more often misinformed -- than informed -- the public about their health. + Mr. Coolidge kept his battle with depression under wraps. White House doctors hid the seriousness of President Wilson's stroke and President Roosevelt's heart condition. President Kennedy denied reports that he had Addison's disease, or adrenal insufficiency, and required hormone injections. President Lyndon B. Johnson chose not to reveal that doctors had removed a basal cell skin cancer, a relatively minor condition, because disclosure would involve the word ''cancer.'' + And the tendency toward secrecy has continued, even in more recent times. + In 1992, former Senator Paul E. Tsongas of Massachusetts, a Democratic presidential candidate, and his doctors said he had been cured of a lymphoma cancer. In fact, Mr. Tsongas had had a recurrence of the disease and died a day short of what would have been the end of his first term had he been elected. + Meanwhile, in 2000, Dick Cheney, then the Republican vice-presidential nominee, and his doctors and aides were sharply criticized for failing to promptly tell the public that he had suffered a mild heart attack. + Even this year's presidential candidates, who speak openly about their illnesses, differ in their comfort levels with the subject. While Mr. Giuliani regularly discusses his cancer at campaign stops, Mr. McCain rarely does so. (Mr. McCain is 70 and is often asked about his health and whether he is too old to be president.) + Mr. Romney discusses his wife's condition when talking to voters, but typically only when asked, according to his campaign staff. (He has been criticized for opposing stem-cell research, which some believe might eventually lead to a cure for multiple sclerosis.) + Senator Joseph R. Biden, the Delaware Democrat and presidential candidate, said friends have urged him to talk more about surviving a life-threatening aneurysm nearly two decades ago. He said they believe it would personalize his campaign and show people ''I wasn't born in a blue suit behind a podium.'' + He mentioned his health scare for the first time a few weeks ago, but has hesitated since because it feels too personal. ''I'm a bit uncomfortable,'' Mr. Biden said of the idea. ''I'm still wrestling with it.'' + The Edwardses, who did not have much choice in publicizing their situation, say they have been overwhelmed by the flood of supportive messages since their announcement. + But such candor may backfire, some warn. Some Democrats believe that Mr. Edwards's decision to continue his presidential bid could pose some risks, particularly should his wife's illness worsen. Political analysts say voters may well conclude that he would be too distracted to hold the presidential office. + In that regard, Mr. Green pointed out, history offers some cautionary lessons. + In the 19th century, President Benjamin Harrison's wife, Caroline, died of tuberculosis during his re-election bid. He was accused of being distracted and inattentive to his campaign, which accounted, in part, for the lopsided election result, according to a brief biography of Mr. Harrison compiled by the University of Virginia. + In 1892, the voters handed the presidency to Mr. Harrison's rival, Grover Cleveland. + It was the most decisive presidential victory in 20 years. + THE NATION + Correction: April 1, 2007, Sunday An article last Sunday about the ways in which presidential candidates deal publicly with illness referred incorrectly to the death of former Senator Paul E. Tsongas of Massachusetts, who had lymphoma cancer. He died on Jan. 18, 1997, two days short of what would have been the end of his first term had he been elected president in 1992 -- not one day short. (The error first appeared on Oct. 3, 2004, in an article about the health of Senator John Kerry.) \ No newline at end of file diff --git a/nyt-2007_small/1836733.xml.txt b/nyt-2007_small/1836733.xml.txt new file mode 100644 index 0000000..aaf0e95 --- /dev/null +++ b/nyt-2007_small/1836733.xml.txt @@ -0,0 +1,14 @@ +The Money Race + + The Money Race + +With the first quarter of the year ending tomorrow, presidential campaigns are preparing to release their fund-raising tallies on April 15. A look at fund-raising efforts by a few of the leading candidates. + JEFF ZELENY + THE DEMOCRATS + Senator Hillary Rodham Clinton With $11 million transferred from her Senate re-election account, the campaign has myriad advantages. The fund-raising list is among the biggest in the party, providing a wide reach to Democratic households across America. Emily's List, a political action committee that raised $45 million last year to elect female candidates, is working on Mrs. Clinton's behalf. Bill Clinton has tirelessly raised money, too, expanding the campaign's reach. + John Edwards As the only candidate in the race who ran for president in 2004, his campaign's fund-raising list remains fresh, but it remains an open question whether his network of trial lawyers will come through as it did four years ago and if he can expand his appeal to avoid falling off in the second quarter. Unlike many others in the field, he can hold fund-raisers without the shackle of the Senate schedule. Will sympathy toward his wife, Elizabeth's, cancer inspire more contributions? + Senator Barack Obama The political equivalent of a popular start-up company, fund-raising events were held across the country for as low as $25 per ticket. The idea is to sign up a new generation of supporters as long-term investors, who could spread the word and keep their contributions coming. A strong grass-roots appeal has developed on the Internet, even while picking off a few former Clinton administration officials and firing up a network of fellow Harvard lawyers. + THE REPUBLICANS + Rudolph W. Giuliani The former New York mayor has sought to keep his fund-raising activities well below the radar, but last week he held events almost every day and expects to have held 57 by the end of the quarter tomorrow night. He has a built-in advantage with New York money and contacts from his two terms as mayor, but is vigorously working to expand his appeal. He starts from scratch, with no federal account to transfer to his presidential account. + Senator John McCain With a fund-raising engine propelled by many of the same forces who financed both campaigns of President Bush, the campaign has been raising money since opening an exploratory committee on Nov. 16. After delaying an official announcement of his presidential campaign, in part to conserve money, Mr. McCain has recently sought to play down its first-quarter standing. Among his techniques: turning public forums into fund-raisers. + Mitt Romney Of all the contributions to his campaign, three sources stand out as distinct: Mormons, Massachusetts and Mr. Romney's connections in the business world from his corporate days. The former Massachusetts governor kicked off his campaign with a fund-raising telethon and has recently encouraged young entrepreneurs to raise money for his campaign. If they find success, 10 percent of the money goes into their pockets. HEADING TOWARD 2008 \ No newline at end of file diff --git a/nyt-2007_small/1837695.xml.txt b/nyt-2007_small/1837695.xml.txt new file mode 100644 index 0000000..729e227 --- /dev/null +++ b/nyt-2007_small/1837695.xml.txt @@ -0,0 +1,9 @@ +Tallying Votes, Not Money + +Now, back to the voters. + As the presidential campaigns calculate their first-quarter financial reports -- working overtime to interpret their results favorably -- the candidates turn their attention back to campaigning and signing up rank-and-file supporters. + On the Republican side, former Mayor Rudolph W. Giuliani of New York makes his inaugural visits to New Hampshire and Iowa. Former Gov. Mitt Romney of Massachusetts fires up his ''Ask Mitt Anything'' forums. And Senator John McCain of Arizona returns from a trip to Iraq. Two new candidates officially in the race -- Tommy G. Thompson, the former Wisconsin governor, and Tom Tancredo, a Colorado congressman -- begin testing their respective messages in early voting states. + Among the Democrats, Senator Hillary Rodham Clinton of New York travels through eastern Iowa. Senator Barack Obama of Illinois talks health care in New Hampshire. And John Edwards of North Carolina returns to Iowa with his wife, Elizabeth, for the first time since she announced that her cancer had returned. Senators Joseph R. Biden of Delaware and Christopher J. Dodd of Connecticut also have full schedules. + This week, during a Congressional spring break, the presidential candidates whose day jobs normally are on Capitol Hill have been given a reprieve from flying back to Washington to cast votes. So the candidates have loaded their schedules with meeting voters in early primary states. But by week's end, virtually all the contenders have the same itinerary etched into their calendar: a vacation day or two for Easter. + JEFF ZELENY + THE CAUCUS \ No newline at end of file diff --git a/nyt-2007_small/1837697.xml.txt b/nyt-2007_small/1837697.xml.txt new file mode 100644 index 0000000..53813e2 --- /dev/null +++ b/nyt-2007_small/1837697.xml.txt @@ -0,0 +1,7 @@ +Big States Weigh In + +Statewide surveys conducted by Quinnipiac University in Florida, Ohio and Pennsylvania reveal that a significant number of voters do not yet have opinions of Senator Barack Obama, Democrat of Illinois, and Mitt Romney, the former Republican governor of Massachusetts. + Democratic voters have a more favorable impression of Senator Hillary Rodham Clinton of New York than they do of John Edwards, the former North Carolina senator. Republican voters have more favorable views of Rudolph W. Giuliani, the former New York mayor, than they do of Senator John McCain of Arizona. + Florida and Pennsylvania have joined the recent rush by states to try moving their primaries earlier in the process; Ohio's primary is set for the first Tuesday in March. + MEGAN THEE + THE CAUCUS: THE POLLS \ No newline at end of file diff --git a/nyt-2007_small/1837803.xml.txt b/nyt-2007_small/1837803.xml.txt new file mode 100644 index 0000000..51656cb --- /dev/null +++ b/nyt-2007_small/1837803.xml.txt @@ -0,0 +1,24 @@ +Romney Leads G.O.P. in Money, Tapping Wall St. and Mormons + + Romney Reaps $20 Million to Top G.O.P. Rivals + +Mitt Romney's presidential campaign said Monday that it had raised $20 million in the first quarter, tapping two distant but rich networks -- Wall Street and the Mormon Church -- to easily outpace his better-known Republican primary rivals. + Senator John McCain, the Arizona lawmaker once considered the front-runner, brought in $12.5 million, his campaign said. It was an unexpected shortfall that could hamper his momentum, and his campaign acknowledged disappointment. Former Mayor Rudolph W. Giuliani's campaign said it had raised about $15 million. + Both of those figures would have set records in previous years, but on Monday, they were dwarfed by the money raised by Mr. Romney and Senator Hillary Rodham Clinton, Democrat of New York, who brought in $26 million. + Kevin Madden, a spokesman for Mr. Romney, said the total was ''indicative of the extraordinary success he has had reaching out and discussing important issues with the American people.'' Mr. Madden credited an online system -- called ''com-Mitt'' -- that the campaign had set up for volunteer fund-raisers to send information and solicitations by e-mail to their friends and associates. + Still, polls show that Mr. Romney remains relatively little known outside Massachusetts, where he was elected governor in 2002; the business world, where he delivered hefty returns for investors in Bain Capital, the private equity firm he founded; and Utah, where he has played a prominent role in the Church of Jesus Christ of Latter-day Saints and helped oversee the 2002 Winter Olympics. + Although Mr. Romney's membership in the Mormon Church has often been discussed as a potential political liability, he has taken deliberate steps to turn his affiliation with the church into a fund-raising asset. He has tapped wealthy Mormons including the Marriott family, founders of the hotel chain, and Jon M. Huntsman Sr., who made a fortune in plastics packaging. + Last year, for example, Mr. Huntsman and his sons gave more than $100,000 to political action committees set up to lay the groundwork for Mr. Romney's campaign. A handful of other Mormons have contributed especially heavily as well. They made the contributions through a chain of federal and state committees Mr. Romney set up that allowed donors to contribute more than the $5,000 limit on gifts to federal PACs.. Residents of Utah, the center of the Mormon Church, contributed about 15 percent of the total contributions, more per capita than any other state. + Last fall, three officials of the Romney campaign met privately with a senior leader of the Mormon Church in Salt Lake City about reaching out to the six million Mormons around the country. The Boston Globe reported that Romney campaign officials also contacted officers of the church's Brigham Young University, where Mr. Romney attended college. Two administrators of the university's business school later sent an e-mail message to 150 supporters and alumni soliciting donations for the campaign, the newspaper reported. + Tax laws bar churches and other tax-exempt charitable groups from partisan politics. The church and the Romney campaign both issued statements attesting to church's neutrality as an institution and denying any effort to enlist it in the Romney campaign. A spokesman for the Mormon Church said that the meeting was a ''courtesy visit,'' and that Brigham Young had said the two deans acted inappropriately. + Mr. Madden said Mr. Romney was ''focusing on building support across the broad spectrum of the American electorate,'' not just among Mormons. + ''Having said that,'' Mr. Madden continued, ''I expect that the way a lot of Greek-Americans supported Michael Dukakis or Jewish voters supported Joe Lieberman, there will be support for Governor Romney among members of the Church of Latter-day Saints.'' + Mr. Romney's campaign also said Monday that he had lent the campaign $2.35 million out of his own pocket as seed money, demonstrating a willingness to open his wallet for campaign money that none of his 2008 rivals can match. Mr. Romney has never disclosed his net worth, but analysts who study compensation at private equity firms say his earnings as the founder of Bain Capital are likely to amount to several hundred million dollars. In 2002, he spent $6 million of his own money on his $9.4 million campaign for governor. + What is more, records of Mr. Romney's political action committees indicate that he also accumulated a valuable Rolodex during his years at Bain. Executives of Bain Capital and its sister firm, Bain Consulting, contributed more than $64,000 over the last two years to Mr. Romney's federal political action committee, Commonwealth PAC, and in 2002 they gave more than $14,000 to his campaign for governor. Executives of Fidelity Investments, the mutual fund giant based in Boston, contributed $30,000 to the PAC and $22,500 to the campaign for governor. Massachusetts campaign laws capped individual contributions at $500. + Mr. Romney helped make many others rich through steep annual returns for investors in Bain Capital and through its payments to finance or buy out private companies. Some. like Thomas Stemberg, founder of Staples, have returned the favor with political contributions. + Meg Whitman, who has made a fortune as chief executive of eBay, met Mr. Romney when she worked as a Bain consultant. In January, she signed on as a financial co-chairwoman of his presidential campaign. She and scores of others called their own contacts on his behalf as part of a public demonstration of his fund-raising prowess that raised more than $6 million in one day. + Unlike the other leading primary campaigns, Mr. Romney's declined to disclose how many individual donors had contributed or how many small donations it had received. Some campaigns use those measurements to show that they have broad support. + Mr. Romney's first-quarter take may undercut some of Mrs. Clinton's triumph, but his lead over his party rivals most damages Mr. McCain. + Mr. McCain's campaign manager, Terry Nelson, said in a statement that the campaign was ''taking the necessary steps to ensure fund-raising success moving forward.'' + A McCain campaign official, speaking anonymously to discuss internal strategy, said its managers had realized a few weeks ago that its fund-raising system needed an overhaul. The campaign has now adopted a system for more clearly tracking the contributions of its surrogate fund-raisers just as the other leading campaigns have done. + THE 2008 CAMPAIGN \ No newline at end of file diff --git a/nyt-2007_small/1838163.xml.txt b/nyt-2007_small/1838163.xml.txt new file mode 100644 index 0000000..fa69d04 --- /dev/null +++ b/nyt-2007_small/1838163.xml.txt @@ -0,0 +1,26 @@ +Early '08 Fund-Raising Has Clear Blue Tint + +For anyone looking for a sign of the health of the Democratic Party going into the 2008 presidential campaign, it came Wednesday with the last of the fund-raising figures reported by the major presidential candidates. + With the $25 million reported by Senator Barack Obama's campaign, closing in on Senator Hillary Rodham Clinton's $26 million, the Democratic presidential candidates collectively outperformed the Republicans, and by a substantial amount: Democrats raised a total of about $78 million, compared with just over $51 million by their rivals, according to preliminary first-quarter figures provided by the campaigns. + That is remarkable because Republicans have historically proved better at collecting contributions. In every presidential primary season since 1976, the top fund-raiser was a Republican. + The new numbers offered what even Republicans described as measurable evidence that Democrats today are more confident about their prospects of winning back the White House, and -- not typically for their party -- satisfied with their candidates. That enthusiasm gap is emerging as one of the early dynamics of the 2008 campaign. + ''The Democrats seem to have a lot more hunger for the White House right now than we do,'' said Scott Reed, who managed the presidential campaign of Bob Dole, a Kansas Republican, in 1996. ''Part of it on the Republican side may be Bush fatigue. But clearly, the Republicans are going to need to get it together on finances if we are going to compete with the likes of Obama or Hillary Clinton. It's a concern.'' + Officials of both parties as well as independent analysts said the figures quantified a trend apparent here and in New Hampshire, where Democratic presidential candidates consistently draw crowds at rallies and house parties. + ''This reflects on the part of Democrats not only their view of the prospects of victory in 2008, but also how they are coming off this rush of the 2006 elections,'' said Costas Panagopoulos, a political science professor at Fordham University. ''In terms of Republican candidates, voters are feeling like they have to choose from a list of who represents the least of all evils.'' + The initial Democratic fund-raising advantage seems to be the next step in a shift that began in 2004, when Democratic presidential candidates, starting with Howard Dean, tapped into the Internet to collect money from first-time contributors. It continued in the 2006 midterm elections when Democratic Congressional campaign committees matched or beat their Republican counterparts, helping Democrats to take control of Congress. + ''This is the continuation of a trend of an energized and engaged Democratic base that I think both Obama and Clinton are tapping into and that other Democrats, at the Congressional and local levels, are going to be able to tap into,'' said David B. Magleby, a political science professor at Brigham Young University. + If there is a bright side here for Republicans, it is that Democrats, particularly Mr. Obama and Mrs. Clinton, have enough money to inflict a considerable amount of damage on their fellow Democrats in the nine months before they settle in here for the Iowa caucuses that lead off the primary voting. + And it is still many financial quarters before the general election. Even Democrats who described themselves as delighted by the results said they had no doubt that the Republican nominee will have more than enough money to compete in the general election. + ''I don't think the Republicans are going to lose the presidential election next year because of a lack of money,'' said David Plouffe, the manager of Mr. Obama's campaign. ''But the Republican might have to, oddly enough, work harder for it.'' + Still, that may prove to be small comfort. The Republican who had been viewed as his party's leading candidate, Senator John McCain of Arizona, raised about half what Mrs. Clinton and Mr. Obama reported raising, and trailed even the $14 million raised by John Edwards, the North Carolina Democrat making his second bid for president. + Steve Grossman, who was chairman of Mr. Dean's campaign and is supporting Mrs. Clinton, said Wednesday that he was struck by the size of the change, recalling that Mr. Dean had raised $50 million for his entire campaign. ''You had two Democrats who raised $50 million between them for the first quarter of '07,'' Mr. Grossman said. + Beyond that, the Republican's top fund-raiser, Mitt Romney of Massachusetts, has trailed in national polls partly because he is not widely known, but also because he has been repeatedly attacked in his own party as changing positions on central issues, like abortion and gay rights. + In this unsettled environment it is no surprise that this year, the Republicans, not the Democrats, seem hungry to find someone else to get into the race, as reflected by the interest drawn by the potential candidacies of the unrelated Thompsons: Fred of Tennessee and Tommy of Wisconsin. + And the release of these figures has offered a statistical basis to support anecdotal evidence that Democrats are more optimistic about winning the White House. + Democrats as a rule have been drawing much bigger crowds than Republicans. By contrast, Republicans show few signs of pulling out of the doldrums that followed their defeat in the November election. Mr. Reed, the former Dole adviser, suggested that that, combined with opposition to the war and Mr. Bush's low popularity ratings, was sapping the energy of voters in his party. + The Democrats' figures left no question that Mrs. Clinton faces a fund-raising equal in Mr. Obama. Any confidence that she or her supporters had expressed about an easy march to the nomination was erased when Mr. Obama posted his numbers on The Chicago Tribune Web site. At the same time, Mr. Edwards trailed significantly behind Mrs. Clinton and Mr. Obama, evidence of just how important it will be for him to score an early win here before the race moves to the broad and expensive national field of the Feb. 5 primaries. + The news might be worse in the so-called second Democratic tier. Senators Joseph R. Biden Jr. of Delaware and Christopher J. Dodd of Connecticut both struggled to raise money, despite their credentials and networks, which might prove troublesome for them as they seek continued support. + As no less an expert than Mr. Dean learned, this kind of fund-raising success, even when it draws new people into the process, does not automatically turn into voter support. + Mr. Magleby, of Brigham Young, said he suspected Republicans were suffering from donor fatigue. In the quarters to come, the prospect of another President Clinton could certainly get Republicans to take out their checkbooks. + Yet as the numbers were tallied and Republicans found themselves staring at a $27 million gap, it was clear that the disparity between the two parties this spring was about more than money. + NEWS ANALYSIS \ No newline at end of file diff --git a/nyt-2007_small/1838871.xml.txt b/nyt-2007_small/1838871.xml.txt new file mode 100644 index 0000000..584a73e --- /dev/null +++ b/nyt-2007_small/1838871.xml.txt @@ -0,0 +1,32 @@ +IDEAS & TRENDS: The Early Birds; About That Political Traffic Jam in Iowa + + About That Political Traffic Jam in Iowa + +REMEMBER all the clatter about Iowa losing some of its oversize influence in the presidential race? Kindly disregard. + For all the tinkering with the 2008 primary schedule, for all the attempts by nearly 20 states to have a greater say in picking the nominee, an unusually early swarm of campaign visits here underscores which state -- for now, at least -- is one of the top priorities. A new route may exist on the road to the White House, but no candidates are daring to discount the familiar path of Iowa. In fact, the state has almost certainly become more influential. + In the last week or so alone, the field of Republican and Democratic hopefuls staged more than 50 public events across Iowa. They opened offices, welcomed hundreds of new staffers and made scores of calls to local political dignitaries, just letting them know they were in the neighborhood. + Senator Hillary Rodham Clinton began her Four Corners of Iowa Tour as the sun was setting the other night on a farm outside Fort Madison, in southeastern Iowa. (Yes, she promised to return soon and tackle the other three.) + Senator Barack Obama, who had just traveled the state's western edge, was back again, this time chugging through northern Iowa. (Yes, that was him sitting in the bowling alley in the town of Ida Grove, catching the end of a Final Four college basketball game last Saturday night.) + And Rudolph W. Giuliani made his Iowa debut. (Yes, he vowed to compete here, saying: ''I'm going to run in Iowa the way I ran in New York City.'') + The momentum gained by a strong finish in Iowa is, according to the candidates' current thinking, simply irreplaceable. Try as they might, campaign strategists say, they fear they will not be able to chart a reliable course for winning the nomination without Iowa. + Here's why: Bypassing Iowa -- and its traditional early-voting partner, New Hampshire -- means that a campaign would have to brave most of the month of January without a raft of free publicity, which comes through endless news coverage. + Ordinarily, it might be worth the risk. But with California, Texas, New York and Illinois among the states looking to set their own primaries in early February, immediately after the first round, the luxury of gaining traction slowly and steadily between contests is gone. (Even Senator John McCain, who thumbed his nose at the ritual of Iowa eight years ago, finds himself as invested in the state today as any candidate.) + Iowa is tentatively scheduled to hold its caucuses on Jan. 20, followed by Nevada, New Hampshire and South Carolina. But the calendar won't be settled until later this year and could be upended if New Hampshire follows through with a threat to shift its primary earlier than Jan. 27. (Note to candidates: Don't make plans for December 2007. New Hampshire's secretary of state, William Gardner, could spoil your holiday by calling an early election.) + The realignment of the political calendar -- if last week's schedule is any guide -- has cranked the political thermostat in Iowa higher than ever this far in advance of the election. + Eight years ago, the presidential campaign was barely purring along in early April. George W. Bush? He had yet to step foot in the state. + ''We're seeing an unprecedented number of candidates come through this state,'' Iowa's governor, Chet Culver, said in an interview. ''We are giving them a chance to meet the voters, take tough questions and share their vision for the future of the country with people who care passionately about the presidential selection process.'' + That is precisely the kind of attention that California's governor, Arnold Schwarzenegger, had in mind when he signed legislation to advance the date of his state's primary. But fund-raising aside, campaign officials say it would be malpractice to focus more than a sliver of their attention right now on California and other states considering a Feb. 5 primary date. The time will come, they said, but only for those who do well in the opening chapters of the campaign or have stockpiled enough money to stay in. + The political dance in Iowa, with the wide-open field on both sides of the ticket, has a far different feel than the contest of 2004. The Des Moines Register, the state's largest daily, is so stuffed with political coverage, it can seem as though January has already arrived. + With the first quarter of fund-raising behind them and a Congressional recess in Washington, a dozen candidates made appearances in every quadrant of the state, from the Missouri River in the west to the Mississippi River in the east, from the Missouri border on the south to the Minnesota border on the north. + Long-shot candidates came, hoping to catch an Iowa tail wind, as Jimmy Carter did in 1976. (He did not win the caucuses, mind you, but came in second -- after ''uncommitted.'') + Tom Tancredo, a Republican congressman from Colorado, formally declared his presidential candidacy here the other morning. He made his pitch during the first hour of the ''Mickelson in the Morning'' talk-radio show on WHO-AM in Des Moines. After the commercial break, Senator Joseph R. Biden, Democrat of Delaware, slid into the studio to take his turn. + A former Wisconsin governor, Tommy Thompson, who has been acquainting himself with Iowa Republicans on many weekend trips this year, held his first rally as an official candidate. Senator Christopher J. Dodd, Democrat of Connecticut, has secured prominent real estate for his headquarters along Grand Avenue in Des Moines. And John Edwards conducted back-to-back-to-back live TV interviews the other night with each station in the state capital. + It would be foolhardy to place much stock in polls at this early date, but the few polls taken so far argue that Mr. Edwards, the only major candidate who competed four years ago, holds an early edge among Democratic caucus goers. (He is, after all, perhaps the only candidate who can often call voters by name.) + As for the Republicans? It is anyone's guess. But voters warn out-of-town reporters not to count out candidates like Senator Sam Brownback of Kansas, a conservative who appeared at two prayer breakfasts in as many days here last week. + Take a listen to nearly all the candidates and it becomes clear that they are becoming well-practiced in their flattery of Iowa. (''We have a lot of farms in New York,'' Mrs. Clinton told her audience at a picnic in Fort Madison. ''Do you all know that? It's true. I know nobody believes that, but I tell everybody that.'') + And, of course, in nearly every trip to the state, most candidates pause long enough to pledge their allegiance to the Iowa caucuses. ''There's nothing that can compare with the one-on-one meetings that you have with the people of Iowa,'' Mitt Romney, the former Massachusetts governor who is seeking the Republican presidential nomination, told reporters at the Iowa Statehouse. + As campaigns pencil in their schedules for the second quarter of the year, strategists for candidates in both parties say that travel to Iowa and New Hampshire is likely to consume about half of their days on the road. That's not to say campaign planes won't be heading to California, Florida, New York, Texas and other states considering an earlier primary next year. + But those trips, at least the public campaign events, are likely to be swift. + Take, for example, a trip Mr. Giuliani took on a recent day to meet voters in Oakland, Calif., where he logged a 24-minute visit, according to the stopwatch of the veteran political writer Carla Marinucci of The San Francisco Chronicle, who described the appearance as ''lightning-quick.'' + In Iowa last week, he lingered for the better part of a day. + The Early Birds \ No newline at end of file diff --git a/nyt-2007_small/1839214.xml.txt b/nyt-2007_small/1839214.xml.txt new file mode 100644 index 0000000..6fc280e --- /dev/null +++ b/nyt-2007_small/1839214.xml.txt @@ -0,0 +1,18 @@ +ON THE CAMPAIGN: Polling Early and Often + +Mitt Romney's campaign fired off an announcement last week, trumpeting a new poll that found Mr. Romney had taken the lead among Republican presidential candidates in New Hampshire. ''Zogby: Romney Rockets!'' it said. + Election day is 19 months away. No matter. The 2008 campaign cycle has been distinguished by a blizzard of polls, being conducted by news organizations and independent (and not-so-independent) groups. They are providing a near-daily measure of who is up and down in the Democratic and Republican primaries and even offering hypothetical match-ups for the general election. + ''There is like a poll every day: I've never seen anything like it,'' said Mark Penn, a pollster who is the chief strategist for the presidential campaign of Senator Hillary Rodham Clinton. ''I joke about it: 'We have four polls today!''' + Polls can be problematic for candidates as they struggle to convince the world that, based on the latest blip or swing in a poll, their campaigns are not collapsing. (Ask Senator John McCain of Arizona about that today). But polls do have their uses for resourceful campaigns which have been quietly seeking to take advantage of this new environment. + The Clinton campaign made an early decision this year to spotlight--in the form of a regular patter of news releases from her war room any poll that showed her ahead. That is something that some campaigns, particularly those of presumed front-runners, often avoid, because it makes it harder to knock down any poll that comes along later that shows you in trouble. (This is known as the live-by-the-poll, die-by-the-poll rule of politics). + In this case, Mrs. Clinton's advisers decided it was worth the risk. They determined (yes, after doing some polling), that her biggest obstacle to winning the nomination was the perception among Democratic primary voters that she could not win. Erasing concerns about her electability by displaying evidence that she was a winner, the thinking went, would go a long way to getting her elected. + For all the candidates, the poll numbers can be like the first-quarter fundraising numbers the campaigns released last week: validators for all the candidates as they shout for a moment of attention from the press, from contributors, from party leaders in these extremely crowded fields. + In Mr. Romney's case, his continued low numbers in the polls as meaningless as that is right now clearly was irking people at his campaign headquarters in Boston. Mr. McCain's aides had taken to poking fun at his anemic poll standing; more than that, it had to be humiliating for Mr. Romney, who is from Massachusetts, to be trailing Mr. McCain and Mr. Giuliani in his own backyard, New Hampshire. The announcement of the New Hampshire poll read like the e-mail equivalent of a sigh of relief. + Campaigns rely primarily on their own internal polling to help understand and track public impressions of their own candidates, their opponents and the issues. But public polls even at this early stage, if of dubious quality when it comes to measuring the pure horserace, can offer telling information about the form and direction of the race as well. + Mr. McCain must be tracking voter attitudes about the war in Iraq as he prepares to give a speech Wednesday arguing that President Bush's policies in Iraq are working. The intensity of concern among voters about issues like health care can shape the substance of the messages that candidates place before the public. Candidates carefully track areas of strength and vulnerability; political strategists say they are watching Senator Barack Obama's standing among African-American voters, and Mrs. Clinton's among women. + ''Voters are a long way from finally deciding that doesn't mean it doesn't matter where they are today,'' Mr. Penn said. + Yet most pollsters themselves will tell you that as a matter of actually predicting how people will vote presumably what these surveys are intended to do the polls are suspect, particularly in a year with so many candidates and no obvious front-runner. + Historically, they have been wide off the mark when it comes to Democrats; they have more accurate for Republicans, but there really hasn't been a Republican contest like this one with the party's presumed front-runner, Mr. McCain, struggling so early -- in a long time. + Voters are being asked to give opinions about candidates who in most if not all cases they do not really know. A national poll is not going to tell you who is going to win the Democratic nomination; the parties don't chose their candidates in national elections. Polls in influential early-voting states like Iowa are particularly unreliable. Trying to divine the winners of hypothetical general election match-ups now is, with all due respect to the news organizations shelling out money to pose those kind of questions, the polling equivalent of a parlor game. + Consider this: as late as December 2003, Howard Dean was being treated as the all-but-certain Democratic nominee, reflected in many polls. His collapse in Iowa began shortly after Jan. 1 when, the holiday season over, most voters began to really begin to examine the credentials of the Democratic field. That is a lesson worth remembering these days. + ''There's value to understanding who the voters are, what they are thinking, whatever,'' Mr. Penn said. ''It's another thing to use these polls for transient headlines,'' he said. ''There's polling abuse out there.'' \ No newline at end of file diff --git a/nyt-2007_small/1839300.xml.txt b/nyt-2007_small/1839300.xml.txt new file mode 100644 index 0000000..14b777b --- /dev/null +++ b/nyt-2007_small/1839300.xml.txt @@ -0,0 +1,8 @@ +FORUM + +MoveOn.org, the liberal online advocacy group, will hold a virtual town hall meeting tonight in which six Democratic presidential candidates will take questions on Iraq from MoveOn members. Afterwards, members will vote for their favorite by e-mail or cellphone text message. + ''It's 'American Idol' meets American politics,'' said Trevor FitzGibbon, a spokesman for MoveOn. + Senators Hillary Rodham Clinton, Barack Obama and Joseph R. Biden Jr., Representative Dennis J. Kucinich, Gov. Bill Richardson and former Senator John Edwards will join in, either live or on videotape. Three Republican candidates -- Senator John McCain, Rudolph W. Giuliani and Mitt Romney -- declined to participate. + The meeting will be broadcast on the group's Web site and on some radio stations that carry Air America programming. MoveOn plans similar events, on health care and energy policy, this spring. + JOHN M. BRODER + THE CAUCUS \ No newline at end of file diff --git a/nyt-2007_small/1840950.xml.txt b/nyt-2007_small/1840950.xml.txt new file mode 100644 index 0000000..1e67d9c --- /dev/null +++ b/nyt-2007_small/1840950.xml.txt @@ -0,0 +1,5 @@ +Starting to Address the Big Questions + +For those who crave a little substance -- impatient with the often who's-up, who's-down nature of the presidential campaign -- the time has come. These days, it is hard to take a step without bumping into a candidate delivering a major policy address and seeking to establish credentials on the big issues. Senator John McCain, Republican of Arizona, talked yesterday in Memphis about his economic policies. A speech by Senator Hillary Rodham Clinton, Democrat of New York, on ''women and leadership'' at Rutgers was postponed because of the weather. Senator Barack Obama, Democrat of Illinois, is speaking about foreign policy today in Chicago, and former Gov. Mitt Romney of Massachusetts will deliver what he bills as a major address tomorrow in Washington. His campaign is keeping the topic a secret. And if it's Thursday, it must be Senator Christopher J. Dodd, Democrat of Connecticut, talking about energy. + ADAM NAGOURNEY + HEADING TOWARD 2008: ISSUES \ No newline at end of file diff --git a/nyt-2007_small/1840995.xml.txt b/nyt-2007_small/1840995.xml.txt new file mode 100644 index 0000000..64faf9d --- /dev/null +++ b/nyt-2007_small/1840995.xml.txt @@ -0,0 +1,17 @@ +Restructuring Of Schools Is Detailed By Chancellor + + Klein Specifies Restructuring of City Schools + +Schools Chancellor Joel I. Klein unveiled new details yesterday about how the city school system will be organized once the 10 regional superintendent offices are abolished in September as part of the Bloomberg administration's latest restructuring of the bureaucracy. + The reorganization is a sort of inversion of the city school administration. Instead of the traditional model in which principals work directly for a superintendent, each of the city's more than 1,400 principals will choose a ''school support organization'' to work with their schools, and will pay these groups out of the school's budget. + ''Until now many educational decisions were made outside of the schools and classrooms,'' Mr. Klein said during a news conference at Education Department headquarters. + Principals will have a menu of choices, at various prices, Mr. Klein said. At the low end, principals will pay $29,500 to join the so-called ''empowerment network,'' in which they are largely freed from oversight in exchange for agreeing to meet performance targets that include higher test scores. + At the high end, schools can choose to contract with the Success for All Foundation, a private nonprofit company based in Baltimore that offers a ''whole school reform'' model at a cost of up to $145,215, depending on enrollment. Smaller schools will be able to contract with the Success for All for as little as $44,694. + While the chancellor maintains that freeing principals from the daily oversight of superintendents will give them more power than ever to raise student achievement, critics have urged the Bloomberg administration to halt its plans, saying that too much is changing too fast and that the bureaucratic reconfiguration will not help children. + Principals are being asked to choose among three options: empowerment, in which schools are organized into networks and led by network support teams; partner support organizations, in which nine private nonprofit groups can be hired on contract to provide support to schools; and four learning support organizations, run by former regional superintendents, each with a different theme. + Officials expect that the learning support organizations, because they are run by veterans of the city school system, are likely to attract the largest number of principals. + Kathleen M. Cashin, now the superintendent of Region 5 in Brooklyn and Queens, is offering a ''Knowledge Network'' group that will help schools impose a ''content rich'' curriculum, focused on crucial facts that students need to know in science, literature, history, the arts and music, at a cost of $42,438 per school. + Laura Rodriguez of Region 2, in the Bronx, is offering the ''Leadership Learning'' support organization, which will focus on strengthening the skills of principals and assistant principals, at a cost of $55,000 per school. + Judith Chin of Region 3 in eastern Queens has created the ''Integrated Curriculum and Instruction'' group, which promises to help schools develop a multidisciplinary ''thinking curriculum,'' at a cost of $47,500 per school. + And Marcia V. Lyles of Region 8 in Brooklyn is offering the ''Community Learning'' support organization, which will focus on partnerships with communities and families. Dr. Lyles has set three price levels: basic for $33,750; premium for $39,850; and elite for $66,675. Failing schools needing the most help will pay the highest price. + The private groups that will be partnership support organizations include the City University of New York and New Visions for Public Schools, a New York City group. \ No newline at end of file diff --git a/nyt-2007_small/1843422.xml.txt b/nyt-2007_small/1843422.xml.txt new file mode 100644 index 0000000..3155338 --- /dev/null +++ b/nyt-2007_small/1843422.xml.txt @@ -0,0 +1,27 @@ +South Carolina, and a Fish Fry, Gain Political Heft on '08 Path + +Seldom has a slice of white bread and a piece of fried fish carried such weighty significance in American politics. + A day after appearing at the first presidential debate of the year, the field of Democratic hopefuls descended Friday evening on a different South Carolina stage: Representative James E. Clyburn's fish fry. It is an annual event that candidates view as an essential ingredient in courting voters here, a critical stop on the road to the primary. + While Iowa and New Hampshire have long played their traditional roles in opening the presidential campaign season, the South Carolina primary, set for Jan. 29, is emerging as a near equal, a last-chance stop for candidates to demonstrate their strength before being thrust into the coast-to-coast battery of early primaries on Feb. 5. + Although it was the Democrats on display here Friday evening at the fish fry, Republican presidential candidates have been no less visible in South Carolina, testimony to the fact that the primary here is shaping up as a pivotal event on both presidential nomination tracks. Senator John McCain, Republican of Arizona, visited this week to announce his candidacy, and Republicans are coming next month for their own debate. + ''It will have more importance than ever,'' said Mr. Clyburn, who is one of the state's most influential politicians, not to mention the No. 3 Democrat in the House. ''This is really the place where almost anything could happen.'' + The South Carolina primary also presents the first significant opportunity for candidates in either party to test how their messages appeal to black voters and Southern constituencies. + It will provide the first clues to these and other questions: Will Bill Clinton's popularity among black voters help Senator Hillary Rodham Clinton, or will Senator Barack Obama's quest to be the first black president have greater resonance? Can Democrats balance their opposition to the Iraq war with a national security message in a state heavily influenced by the military? And which Democrat, if any, has a shot at capturing Southern support in the general election? + ''All of these candidates see South Carolina as a place where they can jump out and surprise some people,'' said Joe Erwin, the chairman of the State Democratic Party. ''This is so new to people here; it's not like Iowa and new Hampshire where it is so embedded.'' + So the Democratic contenders fanned out Friday, campaigning from Greenville to Charleston. But nearly all made certain they were back in the capital before nightfall to attend the Clyburn fish fry. + ''I have been all over the world, I've been to all kinds of political events,'' Mrs. Clinton said, her voice fading as she screamed over the crowd. ''I can honestly say I've never seen anything like Jim Clyburn's fish fry in the world.'' + The event, part carnival and part buffet with 120 loaves of bread and 1,200 pounds of whiting, spilled out of the ground floor of a parking garage on Gervais Street in downtown Columbia. John Edwards of North Carolina hired a high school drum line to escort him into the show, while Senator Christopher J. Dodd of Connecticut exclaimed, ''We're coming back to South Carolina!'' + The political purpose is clear: few candidates dare to miss it, considering they are working to win Mr. Clyburn's endorsement. (No, he said, he has not decided whether he will issue one this year, but he beamed as he stood alongside the candidates on a crowded stage.) + Mr. Clyburn has held the fish fry since 1992. Why? He wants the candidates to mingle with people who do not typically attend political events. He wants them to dance, to eat, even to drink a can of beer. + This year's crowd was so unwieldy that most of the candidates left long before the Electric Slide began, a sure disappointment to the crush of photographers who had visions of capturing them dancing. (Who, after all, could forget Representative Richard A. Gephardt of Missouri showing his moves four years ago?) + But the candidates vowed to be back, including Gov. Bill Richardson of New Mexico, who bluntly declared that his candidacy depended on South Carolina, saying, ''It's going to be hard to win here, but I need to be in the top three or I'm out.'' + Activity on the Republican side is no less intense, in no small part because of the vivid memories of 2000, when George W. Bush's victory here stopped the momentum Mr. McCain had coming out of his victory in the New Hampshire primary. + Mr. McCain came here from New Hampshire this week to formally announce his presidency in what he described as testimony to the importance he attaches to winning this state to capture his party's nomination. He arrived to find evidence of the fight on his hands: Signs for Mitt Romney, the former governor of Massachusetts, were plastered on light poles in Greenville, where he announced his candidacy in front of a thin crowd on a beautiful morning. + This time around, Mr. McCain seems intent not to replicate his past defeat here. + He told reporters that he thought the main reason he lost in 2000 was because Mr. Bush had assembled a superior network of South Carolina supporters, and he said he had made sure to do the same this time. Chief among them is Senator Lindsey Graham -- ''the most popular official in South Carolina,'' Mr. McCain called him in Greenville -- who accompanies Mr. McCain around the state. + As Mr. McCain knows better than his fellow candidates from either party, the South Carolina primary has unique local issues that can overshadow a campaign. + Take, for example, the controversy over the Confederate battle flag, which has not gone away since it plagued him seven years ago when he first said flying the flag was a state issue, then reversed himself. Just this month, Steve Spurrier, the University of South Carolina football coach, called for the flag to be removed from the Statehouse grounds. + Mr. McCain said he did not think the dispute over the flag would trouble his campaign this time, saying, ''That's behind us.'' But he received several reminders that it was not. On his way to a rally at a flour-processing plant in Columbia, he drove past more than one Confederate flag fluttering in the wind. + At the Democratic debate, Mr. Obama was asked whether he agreed with the N.A.A.C.P.'s asking tourist groups and sporting events to stay out of South Carolina until the flag was removed from the front of the Capitol. + ''I think that the Confederate flag should be put in a museum,'' Mr. Obama said. ''That's where it belongs.'' + He changed the subject, though, before saying whether he agreed with the boycott. \ No newline at end of file diff --git a/nyt-2007_small/1844632.xml.txt b/nyt-2007_small/1844632.xml.txt new file mode 100644 index 0000000..bd1aaf4 --- /dev/null +++ b/nyt-2007_small/1844632.xml.txt @@ -0,0 +1,26 @@ +In G.O.P. Debate Today, Which Tack for Giuliani? + +Which Rudolph W. Giuliani will show up tonight in California for the debate among 10 Republican presidential candidates? The hard-hitting prosecutor? Or Mr. Nice Guy? + Mr. Giuliani has played both in his short debate history. Now, with polls showing him as the front-runner of the Republican field, his performance may well be the most-watched tonight. But it has been a decade since Mr. Giuliani last debated as a political candidate, and his two closest challengers, Senator John McCain of Arizona and former Gov. Mitt Romney of Massachusetts, have more recent experience. + Mr. McCain has actually been in a presidential debate. That was in the 2000 primaries against George W. Bush. Mr. McCain, who was reluctant to participate, joined in at the last minute by satellite hookup. It was not a high point for either candidate as they sniped at each other over religion and politics. + Mr. Romney has participated in debates in his races for the Senate in 1994 and for governor in 2002. His 1994 debate, against Senator Edward M. Kennedy, a Democrat, has become a big hit on YouTube.com because it shows him supporting abortion rights and gay rights, which are positions he does not hold now. + Mr. Giuliani's debate history is encapsulated in a relatively brief period, from 1989 to 1997, all when he was running for mayor of New York. + With his background as a trial lawyer and a United States attorney, Mr. Giuliani burst onto the public stage as a tough, crime-busting prosecutor. He started out as an aggressive debater, prone to attacking his opponent, according to a review of transcripts of his debates. But voters found him mean-spirited, and he eventually adopted a softer, more-controlled tone, at least in his debates, especially when he was ahead in the polls. + In his first debate, in September 1989, when he was trying to win the Republican primary for mayor, Mr. Giuliani came out swinging. He and his opponent, Ronald S. Lauder, sparred over who was the real Republican, in part because Mr. Giuliani was also running as the Liberal Party nominee. + At one point, Mr. Giuliani accused Mr. Lauder of not having written a book that bore his name, calling it ''a book you probably didn't read because somebody else wrote it for you.'' The New York Times called the debate ''one of the most vituperative encounters of the campaign.'' + Mr. Giuliani won the primary and went on to face Mayor David N. Dinkins, a Democrat, in the general election. In the first of two debates, Mr. Giuliani foreshadowed a line of attack he has revived today: that victory by his opponent would spell ruin. + ''If we keep going merrily along, this city's going down,'' Mr. Giuliani warned in the 1989 debate, suggesting that the Democrats were not tough enough on crime. The comment was similar to one Mr. Giuliani made last month in New Hampshire, when he said that any Republican elected president would try to stop terrorist attacks, implying that any Democrat would not. + In their second, more free-wheeling debate, Mr. Dinkins picked up a familiar refrain against Mr. Giuliani, saying that New York needed ''a mayor, not a prosecutor.'' Mr. Giuliani shot back that voters wanted ''a mayor who has nothing to fear from a prosecutor.'' (Questions were swirling around Mr. Dinkins at the time for selling stock in a communications company to his son for much less than it was valued.) + Mr. Giuliani added later that New Yorkers considered Mr. Dinkins as ''more of the same, more of the rotten politics that have been dragging us down.'' + Mr. Dinkins won that election in 1989 by two percentage points. But Mr. Giuliani came back for a rematch four years later and won. In a ferocious test of wills, the two candidates managed to avoid any debates, for the first time in more than three decades of mayoral campaigns. + The stalemate developed because Mr. Dinkins wanted a third-party candidate to share the stage and Mr. Giuliani did not. Mr. Giuliani and Mr. Dinkins had been so obstinate about holding debates on their own terms that Newsday portrayed them as two infants in baby bunting with the headline, ''Two Big Babies.'' + After he became mayor in 1993, Mr. Giuliani participated in two more debates, both against Ruth W. Messinger, a Democrat, as he sought, and won, re-election in 1997. + In the first, Mr. Giuliani resurrected his theme that bad times could return to plague the city if Democrats retook City Hall. + But he was comfortably ahead in the polls and largely followed his counselors' advice to hold himself in check. The polls that showed him ahead also showed that voters thought he did not have a ''likable personality.'' So he smiled, he took notes, he shrugged off attacks. + In their final debate, Mr. Giuliani was so pleasant that Ms. Messinger took note: ''We're certainly seeing the nice Rudy Giuliani tonight.'' + But she said he was really a bully. ''His natural instinct and style is to bully, to divide, to throw religious differences into the middle of a political campaign,'' Ms. Messinger said. + Mr. Giuliani responded: ''Why am I a bully? I'm very passionate about the job that I do as mayor. I expect very, very high performance out of people. Sometimes I get impatient when things aren't happening fast enough.'' + ''I think I'm a pretty nice guy,'' he said, to laughter, in conclusion. + In 2000, when Mr. Giuliani was running for the Senate, the political world was anticipating a historic confrontation between him and the Democratic nominee, Hillary Rodham Clinton. But he dropped out of the race before they could meet on stage. + If each wins their party's presidential primary, Mr. Giuliani and Mrs. Clinton could get that chance to debate each other after all. + Correction: May 5, 2007, Saturday An article on Thursday about the tone Rudolph W. Giuliani has taken in political debates misstated the year he became mayor of New York. It was 1994, not 1993. \ No newline at end of file diff --git a/nyt-2007_small/1845580.xml.txt b/nyt-2007_small/1845580.xml.txt new file mode 100644 index 0000000..48f2f45 --- /dev/null +++ b/nyt-2007_small/1845580.xml.txt @@ -0,0 +1,24 @@ +Giuliani, Speaking at the Citadel, Calls for a Bigger Army + + Giuliani, Speaking at the Citadel, Calls for a Bigger Army + +Rudolph W. Giuliani called on Saturday for a large-scale increase in the overall troop strength of the Army and the creation of a special force to specifically handle post-combat operations. + In a speech before the graduating class at the Citadel, Mr. Giuliani, who has stressed an aggressive and muscular foreign policy in his campaign for the Republican presidential nomination, said that he would like to see the Army grow to 582,000 troops from its current 512,000, an even larger increase than President Bush has advocated. + ''I believe that America needs at least 10 new combat brigades above the additions that are already proposed by President Bush and are already in the budget,'' he said. + While Mr. Giuliani did not say the troops were meant specifically to aid the effort in Iraq, the war has placed a heavy burden on the armed forces and, as the war stretches into its fifth year, the Army is increasingly stretched thin. + The two other leading candidates for the Republican nomination -- Senator John McCain of Arizona and former Gov. Mitt Romney of Massachusetts -- have also called for an increase in the size of the Army, as has Senator Hillary Rodham Clinton of New York, who is seeking the Democratic presidential nomination. + Mr. Giuliani did not directly condemn the handling of the war by President Bush, but his call for a new force to handle post-combat efforts implied that the approach in Iraq up until recently has been flawed. + Saturday's commencement address was a preview of a much more detailed foreign policy speech that Mr. Giuliani will make in the near future, his aides said. + Mr. Giuliani called the 438 cadets, all of whom were still in high school when he was mayor of New York City and terrorists attacked the World Trade center in 2001, ''the leaders of the 9/11 generation.'' + As he has done in the past, he questioned the wisdom of Democratic leaders who want to pull out of Iraq, saying they ''counsel defeat.'' + ''Never, ever wave the white flag of defeat in front of those who want to come here and kill you and take away your way of life,'' he said. ''Never.'' + ''The reality is that in this world today, there are terrorists, Islamic radical terrorists, who are planning as we sit here at this graduation, who are planning to come here and kill us,'' he said. + To try to avoid the chaos that has ravaged Iraq, Mr. Giuliani called for a hybrid force whose role would be to provide stabilization and help rebuilding. + ''We also have to think about the constructive role that America plays in combat zones when the fighting is over,'' he said. ''The reality is that America is sometimes faced with a difficult choice. After defeating the enemy as we did in Iraq, after a sudden victory in deposing Saddam Hussein, we have a choice.'' + It was clear, he said, that America had an obligation to stay and work to rebuild countries where it has engaged in military conflict, as opposed to President Bush, who, in the 2000 election, railed against what he called ''nation building.'' + ''It will be difficult,'' he said, ''and will require a new organization of our military and civilian components that are needed to do this -- some kind of hybrid we're going to have to create.'' + As the cadets and 5,000 people gathered to hear the speech applauded, Mr. Giuliani emphasized that he would not be inclined to pull out of the conflicts in Iraq or Afghanistan any time soon. + ''Final victory will take time,'' he said. ''The cold war took years, but we prevailed. And it will happen, and on that day your generation will take its place beside the greatest generations in our nation's history.'' + Thanking the cadets for their service, Mr. Giuliani evoked his own memories of Sept. 11, as well as Pearl Harbor and the period between the first and second World Wars, as examples of when the nation was caught unprepared for military challenges. + He vowed not to allow that to happen again, not to implement what he called ''the peacetime dividend'' when the size of the military is decreased because there is no war or overarching threat. + ''We need a force that can both deter aggression and meet the many challenges that might come our way,'' he said. \ No newline at end of file diff --git a/nyt-2007_small/1845717.xml.txt b/nyt-2007_small/1845717.xml.txt new file mode 100644 index 0000000..284a56e --- /dev/null +++ b/nyt-2007_small/1845717.xml.txt @@ -0,0 +1,24 @@ +All in the Families + + All in the Families + +Before I get to the ''but,'' let me say that Hillary Rodham Clinton would make a terrific president. + She has spent decades wrestling with public policy questions about poverty and health care. She is smart and pragmatic on foreign policy issues. And while it would be tough for a liberal or centrist woman to be elected (it's much easier for a conservative like Margaret Thatcher), the rising Democratic tide increasingly makes her look electable. + But there is another issue at stake, one that goes to the heart of what kind of a nation we are. + If Mrs. Clinton were elected and served two terms, then for seven consecutive presidential terms the White House would have been in the hands of just two families. That's just not the kind of equal-opportunity democracy we aspire to. Maybe we can't make America as egalitarian and fluid as we would like, but we can at least push back against the concentration of power. We can do that in our tax policy, in our education policy -- and in our voting decisions. + The political aristocracy in this country is more fluid than past nobility, and that is how the Clinton family entered it. But the benefit of membership in that aristocracy has probably increased over time, as larger Congressional districts and the rising cost of campaigns make it harder for an unfinanced unknown to rise in politics. + Particularly after George W. Bush rose to the White House partly because he inherited a name and rolodexes of donors from a previous president, we should take a deep breath before replacing one dynasty with another. + America's history is based on a rejection of aristocracy. It's true that in our early years, most of our leaders were wealthy elites -- and, frankly, they did a superior job. But one of our most fateful elections came in 1828, ''the revolution of 1828,'' with the rise of Andrew Jackson. + Jackson, the rough-hewn fighter, a former child soldier, defeated John Quincy Adams, who symbolized all the daintiness, education and sophistication of the aristocracy that had ruled until that time. + John Quincy Adams was the better man (if Andrew Jackson were reading this, he would challenge me to a duel, which proves my point). But Jackson's election was a healthy milestone for our democracy in that it truly opened up American politics. + It would be unhealthy to vote for or against a person solely because of his or her family. But where there is a pool of similar candidates, it seems reasonable to count inherited (and wedded) advantage as one factor -- and to put a thumb on the scales of those who rose on their own. + Granted, Mitt Romney and Al Gore are also children of the American political aristocracy. And I wouldn't argue that Mrs. Clinton should be excluded from consideration -- just that it's reasonable to count as a factor that her family has already lived in the White House. + In South Asia -- Pakistan, India, Bangladesh, Sri Lanka -- it has been common for a spouse or child to inherit a political office. But that always has the feel of a politically immature democracy. It would feel very Sri Lankan if we had a father-son series and a husband-wife series of presidents. + And if Jeb Bush succeeds Hillary Clinton in the White House, I'll flee to Sri Lanka. + There is one important counterargument -- raised, perhaps not surprisingly, by my wife. It is that if our aim is to open up the political system and broaden opportunity, then what better way than to elect a woman? + It's true that the election of a first woman (or black or Hispanic) might well nourish the American political system, just as the election of John Kennedy as the first Catholic did in 1960. But, as in Argentina or Bangladesh, the election of a first woman loses much of its significance if she has enjoyed a political shortcut as a predecessor's wife. + If we really want a presidential dynasty, then that's fine. But we shouldn't back into it without discussion -- for the second time in eight years. + American democracy was diminished in its infancy by the way power was largely held by a narrow nobility, however talented. Do we really want to embrace a similar concentration of power in the 21st century? + I'm holding another contest for poetry about the Iraq war. Details are on my blog, www.nytimes.com/ontheground -- or just send your (short) poem to iraqpoems@gmail.com. + Paul Krugman is on vacation. + Op-Ed Columnist \ No newline at end of file diff --git a/nyt-2007_small/1845887.xml.txt b/nyt-2007_small/1845887.xml.txt new file mode 100644 index 0000000..f76ade5 --- /dev/null +++ b/nyt-2007_small/1845887.xml.txt @@ -0,0 +1,8 @@ +Sounding Presidential at Graduation + +It is cap-and-gown season, and for the presidential candidates, that means the opportunity to spend May weekends delivering speeches at colleges and universities that are carefully selected to fit a strategic purpose. + Mitt Romney, the former governor of Massachusetts and a Republican whose Mormon faith has given some Christian evangelicals pause, chose to deliver a commencement speech on Saturday in Virginia at Regent University, which was founded by the evangelist Pat Robertson. Senator Hillary Rodham Clinton, Democrat of New York, is focusing on black institutions, perhaps a reflection of how she and Senator Barack Obama, Democrat of Illinois, are scrambling for black voters. Last Saturday, she gave the commencement address at Wilberforce University in Ohio. Later this month, she will speak at the historically black Dillard University in New Orleans, where she can address the student body in a city whose devastation she often cites in assailing the Bush administration's response to Hurricane Katrina. + Mr. Obama chose the nation's first primary state for a commencement address. He will speak on May 19 at Southern New Hampshire University. + And where did Rudolph W. Giuliani, the Republican and former New York mayor, announce that he wanted to increase the Army's troop strength by tens of thousands of soldiers? He chose the Citadel military college in South Carolina, above, where his address was warmly received Saturday. + MARC SANTORA + MESSAGE \ No newline at end of file diff --git a/nyt-2007_small/1847192.xml.txt b/nyt-2007_small/1847192.xml.txt new file mode 100644 index 0000000..e523946 --- /dev/null +++ b/nyt-2007_small/1847192.xml.txt @@ -0,0 +1,32 @@ +The Basics; Just 18 Short Months to Go + +This early on, the 2008 presidential race has so many unknown candidates it has dark horseflies on its dark horses. But maybe you're among the fully one-quarter of Americans who, in a CBS News poll last month, said they were actually paying a lot of attention to this stuff. If so, the quiz below should be a snap. Right? + TOM KUNTZ + 1. Here are the lower-tier Republican presidential hopefuls. Match names with faces: + Senator Sam Brownback of Kansas John H. Cox, Chicago businessman Former Gov. Jim Gilmore of Virginia Rep. Duncan Hunter of California Former Gov. Tommy Thompson of Wisconsin Former Gov. Mike Huckabee of Arkansas Rep. Ron Paul of Texas Rep. Tom Tancredo of Colorado + (One point for each correct match -- potential 8 points.) + 2. A lot of people know that Mitt Romney, the Republican who was governor of Massachusetts, is a Mormon, but did you know that six candidates are Roman Catholic? Who from the list below isn't? + A. Senator Christopher J. Dodd, Democrat of Connecticut B. Former Mayor Rudolph W. Giuliani of New York C. Former Senator Mike Gravel, Democrat of Alaska D. Gov. Bill Richardson, Democrat of New Mexico E. Tommy Thompson F. Brownback G. Cox + (Correct answer: 3 points) + 3. Who has been least associated with presidential politics? + A. Paul Tsongas B. Ron Paul C. Pat Paulsen D. RuPaul + (Correct answer: 1 point) + 4. Match the potential health risk with the politician: Senator Barack Obama, Democrat of Illinois; Fred Thompson former Republican senator of Tennessee; Giuliani; Huckabee. + A. Non-Hodgkin's lymphoma B. Adult-onset diabetes C. Prostate cancer D. Past smoking + (One point for each correct match -- potential 4 points) + 5. Match the eyebrow-raising quote with the speaker: Tommy Thompson, Obama, Giuliani, Romney and Yosemite Sam. + A. ''Freedom is about authority. Freedom is about the willingness of every single human being to cede to lawful authority a great deal of discretion about what you do.'' B. ''I'm in the private sector and for the first time in my life I'm earning money. You know, that's sort of part of the Jewish tradition.'' C. ''In case you missed it, this week, there was a tragedy in Kansas. Ten thousand people died, an entire town destroyed.'' D. ''I'm not a big-game hunter. I've made that very clear. I've always been a rodent and rabbit hunter. Small varmints, if you will.'' E. ''Ya better say yer prayers, ya flea-bitten varmint! I'm a-gonna blow ya to smithereenies!'' + (One point for each correct match -- potential 5 points) + 6. Match the politician -- Tommy Thompson, Tancredo, Paul, Gravel -- with the descriptive phrase found on his official Web site: + A. ''No amnesty for illegal immigrants'' B. The ''one exception to the Gang of 535'' on Capitol Hill C. ''The reliable conservative'' D. ''More like Cicero than Quixote'' + (One point for each correct match -- potential 4 points) + 7. Pomp my Ride. Match the candidates with how they say they roll: + Senator Hillary Rodham Clinton, Democrat of New York Former Senator John Edwards, Democrat of North Carolina Senator John McCain, Republican of Arizona Rep. Dennis Kucinich, Democrat of Ohio Giuliani Obama Dodd Romney + A. 2005 Ford Mustang convertible B. ''For security reasons, we drive in Secret Service vehicles but the Service lets us use a Ford hybrid when we're home...'' C. Ford Escape hybrid, a compact S.U.V. D. Chrysler 300C, a full-size sport-luxury sedan E. ''I don't drive, I navigate.'' F. Cadillac CTS, a midsize luxury sedan G. Ford Focus compact + (One point for each correct match: potential 8 points) + 8. Fred Thompson? Tommy Thompson? What's the diff? + A. Fred hasn't declared candidacy. B. Fred, from Tennessee, has a better southern drawl than Hillary Clinton. C. Tommy advocates trimming welfare more than John Edwards. D. Fred spends a lot less money on a trim than John Edwards. E. All of the above, probably + (Correct answer: one point) + Answers: 1) Brownback, G; Cox, H; Gilmore, F; Hunter, A; Thompson, D; Huckabee, E; Paul, C; Tancredo, B; 2) C; 3) D; 4) Obama, D; Thompson, A; Giuliani, C; Huckabee, B; 5) Thompson, B; Obama, C; Giuliani, A; Romney, D; Yosemite Sam, E; 6) Thompson, C; Tancredo, A; Paul, B; Gravel, D; 7) Clinton, B; Edwards, C; McCain, F; Kucinich, G; Giuliani, E; Obama, D; Dodd, C; Romney, A; 8) E + There are a total of 34 possible points. If your score -- or political-obscurity quotient (P.O.Q.) -- is: + 0-9 You never miss an episode of ''Judge Judy.'' 10-18 You have heeded John McCain's admonition to ''Get a life,'' and may actually know who he is. 19-27 You are a C-Span caller with the TV volume up too loud. 28-34 You are a model citizen, political pro or someone with entirely too much time on your hands. \ No newline at end of file diff --git a/nyt-2007_small/1847825.xml.txt b/nyt-2007_small/1847825.xml.txt new file mode 100644 index 0000000..5508420 --- /dev/null +++ b/nyt-2007_small/1847825.xml.txt @@ -0,0 +1,34 @@ +Terror Attack Scenario Exposes Deep Differences Among G.O.P. Hopefuls + +The scenario presented to the 10 Republican presidential candidates was chilling: Three American shopping malls had been bombed, producing scores of casualties. Terrorists with detailed knowledge of another imminent and deadlier attack had been captured and taken to Guantánamo Bay, Cuba. + The question: How far can the authorities go in interrogating the terrorists to get information to avert a fourth attack? + The answers exposed clear differences among the three leading candidates in a debate last night that amounted to the first direct engagement of the Republican presidential campaign and showed them all maneuvering to emphasize their credentials on national security and as reliable conservatives. + Senator John McCain of Arizona, a prisoner of war in Vietnam, said he would not resort to torture because the United States would lose more in world opinion than it would gain in information. + ''When I was in Vietnam, one of the things that sustained us, as we went -- underwent torture ourselves -- is the knowledge that if we had our positions reversed and we were the captors, we would not impose that kind of treatment on them,'' Mr. McCain said. ''It's not about the terrorists, it's about us. It's about what kind of country we are.'' + Former Mayor Rudolph W. Giuliani of New York said he would back ''every method'' short of torture that interrogators could think of because ''I don't want to see another 3,000 people dead in New York or any place else.'' + Former Gov. Mitt Romney of Massachusetts said he would support ''not torture but enhanced interrogation techniques. And taking a tougher line than President Bush and Mr. McCain, who have said they would like to shut down the detention center at Guantánamo Bay, Mr. Romney said he wanted the facility doubled in size. + The session, sponsored by Fox News and the South Carolina Republican Party at the University of South Carolina in Columbia, was the second debate for this field of candidates. It was a far more free-wheeling, and revealing, session than their meeting 12 days ago in Simi Valley, Calif. + Appearing before a boisterous and decidedly conservative crowd, the candidates answered repeated questions about abortion, immigration and government spending as they sought to defend their conservative credentials. + For Mr. Giuliani, it was his support for abortion rights, a position that isolated him on a stage of 10 Republicans. For Mr. McCain, it was his identification with campaign finance and immigration legislation that was opposed by many in this audience. For Mr. Romney, it was his changing positions on abortion rights and gay rights that had made him the subject of suspicion for many Republicans in states like South Carolina. + Mr. Giuliani, trying to move his campaign past a week in which he has tried to convince conservatives that his positions on social issues should not disqualify him from winning the Republican nomination, repeatedly described the election as a referendum on Republican policies against terrorism, as he reminded an audience of what he had done in New York after Sept. 11. At one point, one of Mr. Giuliani's lesser-known opponents, Representative Ron Paul of Texas, gave what turned out to be a big platform to Mr. Giuliani when he appeared to suggest that the United States invited the attacks of Sept. 11 by having originally invaded Iraq. + ''May I comment on that?'' Mr. Giuliani said, looking grim. ''That's really an extraordinary statement. That's an extraordinary statement, as someone who lived through the attack of Sept. 11, that we invited the attack because we were attacking Iraq. I don't think I've heard that before, and I've heard some pretty absurd explanations for Sept. 11.'' + Mr. Giuliani was interrupted by cheers and applause. ''And I would ask the congressman to withdraw that comment and tell us that he didn't really mean that,'' he said. + All the candidates once again offered strong affirmations of their support of the war in Iraq, reflecting the general consensus among them. ''We must succeed, and we cannot fail, and I will be the last man standing if necessary,'' Mr. McCain said. + As they went through the doomsday terrorist scenario presented by the moderators, the candidates did not offer precise definitions of what they considered terrorism, though their nuanced responses suggested differences about how far they thought it was appropriate for interrogators to go to avert a terrorist attack. + Enhanced interrogation techniques, which Mr. Romney said he would support, refer to methods outside those allowed by the Army's code of justice or the Geneva Conventions. The most publicly discussed technique that has reportedly been used on terrorism suspects involves what is known as water-boarding, where a prisoner is strapped down, head beneath his feet, as water is poured repeatedly on a cloth covering the mouth until the person thinks he is about to drown. + The moderators from Fox News allowed and at times encouraged the candidates to mix it up among themselves. + Mr. Romney sought to skewer Mr. McCain by association, noting his sponsorship of two bills that were particularly unpopular among conservative Republicans: an immigration bill that he was negotiating with Senator Edward M. Kennedy and the landmark campaign finance bill he drafted with Senator Russ Feingold, the Wisconsin Democrat. + ''My fear is that McCain-Kennedy would do to immigration what McCain-Feingold has done to campaign finance and money in politics, and that's bad,'' Mr. Romney said. + Mr. McCain responded with a reference to criticism of Mr. Romney for switching positions on issues like abortion and gay rights as he has moved from the political arena of Massachusetts, where he ran for governor and senator, to running for president. ''Well, I've taken and kept a consistent position on campaign finance reform,'' Mr. McCain said in response to Mr. Romney. ''I have kept a consistent position on right to life. And I haven't changed my position even on even-numbered years or have changed because of the different offices that I may be running for.'' + On several occasions, the candidates sought to divert problematic questions by trying to turn their attacks on Democrats. Mike Huckabee, the former governor of Arkansas, invoked John Edwards, the former senator from North Carolina, as part of an overall wave of criticism by this Republican field of Congress and the White House for spending too much money. + ''We've had a Congress that has spent money like Edwards at a beauty shop,'' Mr. Huckabee said to roars of laughter at the allusion to Mr. Edwards's paying $400 for a haircut. + When Mr. Giuliani was asked how conservatives could accept a candidate who was pro-abortion rights, gay rights and gun control, and who endorsed Mario M. Cuomo, a Democrat, for New York governor in 1994, he responded by invoking quotations by Senator Hillary Rodham Clinton of New York, displaying her as an advocate of a big, activist government. For that, Mr. Giuliani was chastised by one of the questioners, Chris Wallace. + ''Mayor Giuliani, I'm going to give you another 30 seconds to actually answer my question,'' Mr. Wallace said to laughter and applause. + The differences on abortion in the field were illuminated when Mr. Giuliani was asked to respond to a remark by Mr. Huckabee that saying he thought abortion was immoral, but that it should still be legal, was akin to saying, ''I hate slavery but people can go ahead and practice it.'' + Mr. Giuliani responded: ''Well, there is no circumstances under which I could possibly imagine anyone choosing slavery or supporting slavery. There are people, millions and millions of Americans, who are as of good conscience as we are, who make a different choice about abortion. And I think in a country where you want to keep government out of people's lives, or government out of people's lives from the point of view of coercion, you have to respect that.'' + Mr. Huckabee responded by praising Mr. Giuliani for being ''honest about his position, and I think that's a healthy thing for our party and for this debate.'' + But, he added, ''Now, if something is morally wrong, let's oppose it.'' Some of the leading candidates found their conservative credentials challenged not only by the moderators, but also by some of the lesser-known candidates in the field. Representative Tom Tancredo of Colorado said he had witnessed, in the course of his campaign, candidates move to more conservative positions on guns, abortion and immigration. ''You know it's beginning to sound like a Baptist tent revival meeting here,'' Mr. Tancredo said. ''And I'm glad to see these conversions. But I must tell you, I trust those conversions when they happen on the road to Damascus and not on the road to Des Moines.'' + And James Gilmore, the former governor of Virginia, recalled the debate these candidates had in California nearly two weeks ago and said: ''I think that some of the people on this stage were very liberal in characterizing themselves as conservatives, particularly on the issues of abortion and taxes and health care.'' + Their agreement to appear here reflected the state's importance in the party's nominating process: South Carolina is the third state to hold a nominating contest. This was the state that effectively ended Mr. McCain's bid for the presidency in 2000 when he lost badly here, after beating President Bush in the New Hampshire primary. + The state's Republican electorate is overwhelmingly conservative and includes a large number of evangelical Christians. The death of the Rev. Jerry Falwell, the founder of the Moral Majority, prompted a rush of statements from Republican candidates paying tribute to his life, which also served as a reminder of the importance of conservative religious voters in Southern states. + Correction: May 17, 2007, Thursday An article in some copies yesterday about the debate by the 10 Republican presidential candidates at the University of South Carolina in Columbia referred incorrectly to support for the war in Iraq among the candidates. One of them, Representative Ron Paul of Texas, opposes it; they do not all support the war. The article also misidentified the home state of another candidate, Representative Tom Tancredo, in some copies. It is Colorado, not Arizona. \ No newline at end of file diff --git a/nyt-2007_small/1848002.xml.txt b/nyt-2007_small/1848002.xml.txt new file mode 100644 index 0000000..40a3a87 --- /dev/null +++ b/nyt-2007_small/1848002.xml.txt @@ -0,0 +1,23 @@ +Reports Show Wealth as a Common Factor Among 2008 Contenders + +Former Mayor Rudolph W. Giuliani of New York, who just six years ago told a divorce court he had only $7,000 in assets under his control, has amassed a net worth of more than $30 million, much of it from paid speeches. + Former Senator John Edwards of North Carolina, who has spent the last three years crusading against poverty, also reported about $30 million in assets. His income included nearly half a million dollars for advising an elite investment fund and $40,000 for directing a poverty studies program at the University of North Carolina, Chapel Hill. + Mr. Giuliani and Mr. Edwards reported their assets in personal financial disclosure forms released Wednesday by the Federal Election Commission. Federal election laws require all the presidential candidates to file the forms. Together, they offer a glimpse of the general affluence of all the primary candidates and the truly extreme wealth of a few. + Neither Mr. Giuliani, a Republican, nor Mr. Edwards, a Democrat, is the richest of the White House hopefuls. That title belongs to Mitt Romney, a founder of the private equity firm Bain Capital, who has said he expects to report as much as $350 million in assets, including a trust for his heirs. Mr. Romney and Senator John McCain of Arizona, both Republicans, and Senator Hillary Rodham Clinton, Democrat of New York, have all received extensions of the filing deadline for their forms. + Mr. Giuliani's $30 million fortune is the most unexpected information to emerge from the disclosure forms so far. During his divorce from Donna Hanover in June 2001, a lawyer for Mr. Giuliani said he had only $7,000 in personal money ''under his control.'' His salary as mayor of New York at the time was about $195,000, and his local financial disclosure forms showed less than $800,000 in deferred compensation, pension, retirement and mutual funds. He had also signed a contract to write two books for an advance of $3 million. + The latest disclosure form suggests that his biggest source of income was speeches, capitalizing on his celebrity after the terrorist attacks of Sept. 11, 2001. For the period covered by the form -- roughly the calendar year 2006 -- Mr. Giuliani reported making 124 speeches for as much as $200,000 each and earning a total of about $11.4 million. + Self-help and motivational rallies were his top audiences. He made about 26 speeches at events staged by Get Motivated Seminars, the company of the impresario Zig Ziglar, and 8 more at major events put on by the executive education group HSM. + Mr. Giuliani put a value of $5 million to $25 million on his stake in his consulting firm, Giuliani & Company, which he said paid him about $4.1 million last year. He said he received about $1.2 million in income from his law firm, Bracewell & Giuliani. Mr. Giuliani, who opened a New York office for what had been a Texas firm, said the $1.2 million he received reflected a guaranteed payment of $1 million a year from the firm, plus 7.5 percent of the New York office's revenue. + Mr. Giuliani's book has apparently sold well enough to earn royalties covering his $3 million advance; he reported $146,092 in book royalties in 2006. In addition, Mr. Giuliani reported $496 in theatrical royalties. A spokesman for his campaign said Mr. Giuliani earned that money for playing himself in the films ''Anger Management'' and ''The Out-of-Towners,'' as well as for guest appearances on the television programs ''Law & Order'' and ''Saturday Night Live.'' + Mr. Edwards, who earned his millions as a trial lawyer before running for the Senate in 1998, reported assets worth $14.3 million to $44.7 million in 2004. Since leaving office that year, he has devoted most of his energy to promoting efforts to help the poor, but he has recently faced questions about signing on part time in 2005 as an adviser to the hedge fund manager Fortress Investment Group. + Like many hedge funds -- loosely regulated investment companies open only to the rich -- some Fortress funds are incorporated in the Cayman Islands to avoid taxes. Fortress has invested in a firm that lent high-interest rate mortgages to low-income homeowners. A recent wave of foreclosures on such so-called subprime mortgages has elicited accusations from Mr. Edwards and others that some firms had engaged in predatory lending. Mr. Edwards has said he was not involved in any specific Fortress investments in that business. + Mr. Edwards's financial disclosure form shows that he received $479,512 from Fortress in 2006; the forms did not cover 2005. In addition, the forms show that Mr. Edwards sold several million dollars in other assets to personally invest in Fortress funds. His disclosure form put the value of his holdings in Fortress funds at $11.2 million to $24.7 million. + Mr. Edwards has said he joined Fortress both to make money and to learn about finance. + Mr. Edwards's campaign said he gave $350,000 to charities in 2006, including $333,334 in book royalties. + Eric Schultz, a spokesman for Mr. Edwards, said: ''The bottom line is, if you look at where John Edwards comes from and his record, it's clear what makes him tick: helping those who haven't been as blessed as he has been. John Edwards is running for president to give every American the opportunities that he's had.'' + Senator Christopher J. Dodd of Connecticut, another Democratic candidate, also disclosed an investment of more than $50,000 in the Fortress Investment Group. Mr. Dodd reported total assets of more than $1.5 million and a cottage in County Galway, Ireland, that he valued at $100,000 to $250,000. + Not all the candidates were so flush. Representative Duncan Hunter, Republican of California, reported assets worth hundreds of thousands of dollars but also said he owed more than $30,000 in car loans and more than $75,000 in credit card debt. His was the only disclosure form to be filled out in handwriting. + Gov. Bill Richardson of New Mexico, a Democrat, reported owning more than $100,000 in stock and options worth at least $250,000 in a major oil refiner on whose board he once sat. + Former Gov. Mike Huckabee of Arkansas, a Republican, reported assets valued at less than $700,000, including his retirement benefits as a former Southern Baptist minister. He earned about $150,000 in royalties from his book about his weight loss. + Representative Ron Paul of Texas, a libertarian-minded Republican who often warns that excessive government threatens the economy, has put his pessimism into his portfolio. If the dollar collapses, Mr. Paul will be ready: his favorite investments are real estate, silver and gold. + THE 2008 CAMPAIGN \ No newline at end of file diff --git a/nyt-2007_small/1848017.xml.txt b/nyt-2007_small/1848017.xml.txt new file mode 100644 index 0000000..7e7ce09 --- /dev/null +++ b/nyt-2007_small/1848017.xml.txt @@ -0,0 +1,28 @@ +For '08 Résumés, Don't Ask Them To Fill In Blanks + +Stealing a page from the Soviet playbook, the current crop of presidential candidates has taken to eliminating whole chapters of their histories. + Senator Hillary Rodham Clinton's turbulent final years as first lady? While Mrs. Clinton, a New York Democrat, frequently invokes husband Bill on the stump, she has managed to avoid any mention of his impeachment and the unpleasantness leading to it. + Senator John McCain, Republican of Arizona, almost never brings up campaign finance overhaul, perhaps his signature achievement in the Senate. The McCain-Feingold finance law is loathed by many of the conservatives Mr. McCain is courting, and he typically only discusses the measure when opponents hurl it at him -- as Mitt Romney, the former Massachusetts governor, did in a debate on Tuesday. + For his part, Mr. Romney likes to promote his experience as a governor, but is often coy about the state he governed. (Hint: It is viewed by many Republicans as an outpost of run-amok liberalism.) In campaign advertisements in early primary states, Mr. Romney boasts that he was ''the Republican governor who turned around a Democratic state'' and ''vetoed hundreds of spending appropriations.'' But you would never know where. + Didn't John Edwards once run for vice president? Mr. Edwards, a Democrat and former senator from North Carolina, tends to erase his stint as What's His Name's running mate in 2004. + It is no revelation that campaigns conspicuously omit things. There are always unpleasant facts, episodes or viewpoints that run counter to the public self a candidate is marketing. But one of the striking features of the 2008 campaigns is the pungency of the various elephants in the various rooms. Candidates are strenuously de-emphasizing or ignoring completely experiences that are defining and, in many cases, extremely well known. + ''There's always a tension between what can be said, what should be said and what must be said,'' said Edward Widmer, a historian at Brown University who was speechwriter for Mr. Clinton. ''The first candidate to calibrate this tension may move to the head of the pack.'' + In recent days, Rudolph W. Giuliani, the former New York mayor and a Republican candidate, has spoken forcefully of his support of abortion rights, something that placed him at odds with many in his party, and something he spoke little about until recently. + Still unspoken, for the most part is Mr. Giuliani's delicate family situation. His campaign Web site includes nothing about his children, with whom he reportedly has strained relations. They are, in effect, airbrushed from ''Rudy's Story'' (the heading of the biographical section on the Web site). + While Mr. Giuliani has embraced his New York identity, Mr. Romney has effectively run screaming from Massachusetts, a prime breeding ground for presidential also-rans -- Senator John Kerry, Michael J. Dukakis, Paul E. Tsongas and Senator Edward M. Kennedy, among others. + The state is also a recurring villain among Republicans, a view distilled in a wisecrack by a former House majority leader, Dick Armey of Texas, when Democrats announced that their 2004 convention would be held in Boston. + ''If I were a Democrat,'' Mr. Armey said, ''I would feel a heck of a lot more comfortable in Boston than, say, America.'' + When Mr. Romney does mention Massachusetts, it is hardly with native pride: Responding to a question during the debate, he referred to his home of almost 40 years as ''that very difficult state'' and ''the toughest of states.'' + In an internal Romney campaign memorandum obtained by The Boston Globe in February, Massachusetts is listed as a potentially effective ''bogeyman'' for Mr. Romney (along with ''European-style socialism,'' ''Jihadism'' and ''Hillary Clinton.'') + ''Romney is trying to say that he foiled a robbery in a brothel, the brothel being Massachusetts,'' said Ralph Whitehead, a political analyst at the University of Massachusetts in Amherst. ''But the question people will ask is, What was he doing in the brothel in the first place?'' + Mr. Romney chose to formally announce his candidacy in Michigan. He grew up there. His father was governor there. He spoke emotionally about the place (''We love Michigan'') and of how he had always dreamed of coming back. + In a speech last month, Mr. Romney waxed poetic about a recent trip to his summer home. In New Hampshire. + When uncomfortable topics do emerge, campaigns can become touchy, underscoring the extent to which they have rendered the subject taboo. + Mrs. Clinton's campaign did not respond kindly, for instance, when David Geffen, a supporter of Senator Barack Obama, Democrat of Illinois, made critical comments to the columnist Maureen Dowd of The New York Times about the former first couple, with allusions to Mr. Clinton's ''reckless'' conduct. A spokesman for the former first lady promptly called on Mr. Obama to disavow Mr. Geffen; he demurred, and the elephant that is the Clinton marital history receded, for now. + Mr. Obama has presented himself as a fresh face, not steeped in Washington and the proverbial ''politics as usual.'' It is, to be sure, a cornerstone of his appeal, but also an effort to turn what many could see as a potential handicap -- his inexperience -- into an asset. + Mr. Widmer, the Brown historian, says that owning up to a perceived shortcoming can ''provide a healthy exhalation'' for a candidate, ''if a politician's exhalation can be said to improve air quality.'' + Just as he rarely talks about his vice-presidential campaign, Mr. Edwards can be equally reticent about his time in the Senate. He has promoted himself in this campaign as a Washington outsider, an antiwar, antipoverty crusader. (Elephants crossing on Little Guy Street: Mr. Edwards's eight-figure wealth, gigantic home and $400 haircut.) + Others have been haunted by their legislative history. Mr. McCain has been attacked over the McCain-Feingold law by a host of Republicans, including Mr. Romney, during both candidates' debates. + In an interview with Chris Wallace of Fox News last month, Mr. McCain characterized campaign finance as a Beltway issue. + ''Outside of Washington, I never have anybody stand up and talk about McCain-Feingold,'' he said. ''There's nobody who ever does.'' Himself included. + THE 2008 CAMPAIGN \ No newline at end of file diff --git a/nyt-2007_small/1848102.xml.txt b/nyt-2007_small/1848102.xml.txt new file mode 100644 index 0000000..ed01708 --- /dev/null +++ b/nyt-2007_small/1848102.xml.txt @@ -0,0 +1,18 @@ +Going Like 60 (Tick Tick Tick) + +George J. Tenet accuses the White House of bungling the Iraq war and setting him up (''slam dunk'') to take the fall. John McCain says he shopped peacefully in a Baghdad market, then cameras pull back to show him in body armor surrounded by armed guards. John and Elizabeth Edwards explain why her cancer won't stop his run for the presidency. In a prison interview, L. Dennis Kozlowski, Tyco's former C.E.O., answers questions about his $6,000 shower curtain. + Those ''60 Minutes'' reports were some of the more notable moments in television news this season, and all the more surprising for being on a stately news program that until recently seemed slated for extinction. Its creator, Don Hewitt, retired in 2004; some of the most famous television reporters, people like Ed Bradley and Mike Wallace, were dying, retiring or taking emeritus status. News cycles keep getting shorter and more Internet-driven, and younger viewers are fleeing network news in droves. Yet oddly enough, with many new faces and a lot less fanfare, ''60 Minutes'' is quietly flourishing. + One of the paradoxes of the times is that while private lives of public figures have never been more examined, television interviews with politicians seem more and more insipid. Audiences dislike journalists as much as or more than elected officials, and that leaves reporters wary of being too pushy on camera. Its easier for putative news programs like ''Dateline'' to trap sexual predators and examine the content of Anna Nicole Smith's stomach than to hold a sustained give-and-take with Vice President Dick Cheney or Senator Hillary Rodham Clinton. + ''60 Minutes'' has tried to hold onto its tradition of polite but discomfiting inquiry. + Closing in on its 40th year on the air, this CBS Sunday night newsmagazine is to television news what ''Law & Order'' is to procedural drama and ''Saturday Night Live'' is to comedy: proof that no star is irreplaceable and even series that seem exhausted or obsolete sometimes get a second -- or sixth -- wind. + The turnaround is only barely perceptible in the ratings. At the moment ''60 Minutes'' is averaging 13.2 million viewers, a million more than last year, but that increase merely brings the series back to where it was in 2003. + In 1991, when it was still riding a 23-season streak in the Top 10, ''60 Minutes'' drew 26.5 million viewers. Its heyday may be over, but it still has a large audience. So does ''Nightline,'' two years after the departure of Ted Koppel. Many younger viewers say they get all their news from ''The Daily Show With Jon Stewart,'' but millions of people still listen to National Public Radio every morning and regularly watch ''Frontline'' and ''Meet the Press.'' + Hard news, reported accurately and presented with intelligence and style, hasn't entirely disappeared; it just is too often eclipsed by the cacophony of celebrity-stalking entertainment shows and local news programs driven by weather, traffic and dieting tips. + Trying to adapt to the times, ''60 Minutes'' has recruited some youth and glamour: CNN's Anderson Cooper is a contributor this season, and so is Lara Logan, CBS's chief foreign correspondent. The CBS anchor Katie Couric is perhaps the program's biggest star. But at heart, ''60 Minutes'' is still as stodgy and familiar as the tick-tick-tick of the Tag Heuer stopwatch. The strongest on-air personality at the moment is one of the program's blander faces: Scott Pelley, a former White House correspondent and 17-year veteran of CBS News. + Mr. Pelley has the square jaw; even, all-American features; and wavy hair of yesteryear's local anchor, but he turns out to be a calmly compelling interviewer. He goaded Mr. Tenet, the former director of central intelligence, into almost Captain Queeg-like disquisition about Iraq, the C.I.A., torture and how he was mistreated by the president's top advisers. Mr. Pelley and a ''60 Minutes'' crew were with Senator McCain when this Republican presidential candidate took his infamous maximum-security shopping trip in Baghdad. + Mr. Pelley took a skeptical tone with President Bush back in January when Mr. Bush announced that he was deploying more troops in Iraq, but like many other reporters, he refrained from challenging the president's assertions too overtly. He asked Mr. Bush how he felt after learning that there were no weapons of mass destruction after all; the president replied, ''The first thing I did was I put a commission together to take a good, hard look at what did go wrong.'' + Mr. Pelley didn't counter that. Actually the White House fiercely resisted an independent inquiry into intelligence failures and gave in only after intense lobbying by the families of 9/11 victims. + Mr. Pelley is no Mike Wallace, but then again, neither is Mr. Wallace. Now a correspondent emeritus, Mr. Wallace recently interviewed the Republican presidential contender Mitt Romney at home with his wife, Ann, and their large, attractive family. Mr. Romney, a Mormon who supported abortion rights when he was governor of Massachussetts, told ''60 Minutes'' that he changed his mind after studying the ethics of embryonic stem cell research. Mrs. Romney received a diagnosis of multiple sclerosis in 1998. The likes of the National Multiple Sclerosis Society and Nancy Reagan argue that embryonic stem cell research might help provide a cure for that disease as well as for many others. Mr. Wallace didn't ask Mr. Romney or his wife about that. + One reason could be that nowadays viewers judge reporters as harshly as their subjects -- in some cases more. Ms. Couric was criticized for coolly addressing the prospect of death with Elizabeth Edwards, even though Mrs. Edwards made it clear that she wanted neither pity nor gratuitous displays of emotion. ''I had somebody hug me in the last hour with a tear going down the side of her face,'' Mrs. Edwards said. ''And I'm actually hoping that's one of the things that this discussion will fix.'' + Many hosts of major news programs no longer even try to ask pertinent follow-up questions to politicians and government officials. ''60 Minutes'' may not always be as pestering as it once was, but it has found new relevancy by being better than most. + THE TV WATCH \ No newline at end of file diff --git a/nyt-2007_small/1848238.xml.txt b/nyt-2007_small/1848238.xml.txt new file mode 100644 index 0000000..c2096b4 --- /dev/null +++ b/nyt-2007_small/1848238.xml.txt @@ -0,0 +1,41 @@ +Senators in Bipartisan Deal On Broad Immigration Bill + + Senators in Bipartisan Deal on Immigration Bill + +Senate negotiators from both parties announced Thursday that they had reached agreement on a comprehensive immigration bill that would offer legal status to most of the nation's 12 million illegal immigrants while also toughening border security. + If the bill becomes law, it would result in the biggest changes in immigration law and policy in more than 20 years. That would provide President Bush with a political lift and a tangible accomplishment for his second term. It would also be a legislative achievement for the new Democratic leaders in Congress, though they said they would seek changes in the measure. + At the heart of the bill is a significant political trade-off. Democrats got a legalization program, which they have sought for many years. Republicans got a new ''merit-based system of immigration,'' intended to make the United States more competitive in a global economy. + But the politics of the deal are precarious. Democrats are already trying to tamp down concerns of Hispanic groups, who fear that the bill would make it more difficult for immigrants to bring relatives from abroad. At the same time, Republican negotiators face blistering criticism from some conservatives, who say the bill would grant a virtual amnesty to people who had broken the law. + Mr. Bush praised the Senate measure, which incorporates many of his ideas, saying, ''I really am anxious to sign a comprehensive immigration bill as soon as I possibly can.'' + The bill goes next week to the Senate floor, where negotiators predicted that it would receive overwhelming support. One reason for that optimism was the partnership in evidence at the news conference where the package was announced by 10 senators, including Edward M. Kennedy of Massachusetts, a liberal Democrat, and Jon Kyl of Arizona, a conservative who is chairman of the Senate Republican Conference. + Senator John McCain, Republican of Arizona, came off the presidential campaign trail to embrace the compromise, a potentially risky step because the proposal is unpopular with many conservatives, who are expected to play a large role in choosing the party's presidential candidate. + The measure's prospects are less clear in the House, which plans to take up immigration in July. + Representative Rahm Emanuel of Illinois, chairman of the House Democratic Caucus, said, ''Unless the White House produces 60 or 70 Republican votes in the House, it will be difficult to pass an immigration bill similar to the Senate proposal.'' + Representative Xavier Becerra, Democrat of California and a former chairman of the Congressional Hispanic Caucus, said he had grave concerns about the Senate bill. + ''It's a pretty radical shift to go to an employment-based visa system as opposed to a family system,'' Mr. Becerra said in an interview. ''You will continue to have close family members separated from their loved ones because of this policy.'' + The bill includes a temporary-worker program, under which 400,000 to 600,000 foreign workers could be admitted to the country each year. + Mr. Becerra said the proposal would create ''a permanent underclass of imported workers to fill American jobs.'' + The Senate majority leader, Harry Reid of Nevada, offered cautionary words as well, saying: ''I have serious concerns about some aspects of this proposal, including the structure of its temporary-worker program and undue limitations on family immigration. We need to improve the bill as it moves through the legislative process.'' + Mr. Kennedy has acknowledged that the agreement is not the immigration bill he would have written in another political environment. But on Thursday he said, ''The agreement is the best possible chance we will have in years to secure our borders and bring millions of people out of the shadows and into the sunshine of America.'' + Besides creating a path toward legalization for illegal immigrants, the bill would strengthen the border through the addition of more fencing and other security measures and an increase in the number of Border Patrol agents. + The deal sets the stage for a rare victory for Mr. Bush, who set a goal of establishing a new immigration system at the start of his presidency but saw it stymied by his own party. + As the governor of Texas, Mr. Bush had seen firsthand the challenges of border security and the lengths to which impoverished Mexicans were willing to go to enter this country illegally. What he depicted as ''a rational immigration system'' -- one that would offer a temporary-worker program and a way for those who have set up working lives here illegally to become citizens -- was a major part of his ''compassionate conservative'' agenda. + But the Sept. 11 attacks derailed his plan, and by the time he set out to enact it in his second term conservatives were livid over what they called deplorably inadequate efforts to secure the border. That anger, repeated nightly on talk radio and by the CNN host Lou Dobbs, remains, and is seen within the Republican Party as a motivating force for conservative voters in the next presidential election. + As soon as the agreement was announced, players on both sides of the immigration issue rolled out their defense and their offense. + Tony Snow, the White House press secretary, defended the proposal in a television appearance on ''Lou Dobbs Tonight,'' whose host has become one of the most vocal critics of Mr. Bush's immigration policy. + Mr. Dobbs opened the program by calling the deal an apparent victory for ''the pro-illegal-alien lobby.'' The administration was ''hellbent on creating a North American union without the consent of the American people,'' he said, and the plan could ''threaten national sovereignty and security as well.'' + John J. Sweeney, president of the A.F.L.-C.I.O., denounced the bill from a different angle, saying it would create ''a massive guest worker program.'' + ''All workers will suffer because employers will have available a ready pool of labor they can exploit to drive down wages, benefits, health and safety protections and other workplace standards,'' Mr. Sweeney said. + Senator Byron L. Dorgan, Democrat of North Dakota, said he would offer an amendment to eliminate the guest worker program from the bill. + Under the merit-based system envisioned in the bill, the government would adopt a point system to evaluate the qualifications of many people seeking permission to immigrate. Points would be awarded for job skills, education and English language proficiency. + Negotiators emphasized that foreign-born spouses and minor children of American citizens would continue to receive preference in the allocation of visas. + Moreover, they said, family ties would be an advantage in the proposed point system. If two applicants had the same skills and the same educational credentials, but one also had relatives in the United States, that person would receive the visa. + The negotiators insisted that the legalization program was different from amnesty because illegal immigrants would have to pay fines and go through background checks. They could work in the United States under probationary status and could receive renewable four-year ''Z visas.'' Heads of households would have to return to their home countries to apply for green cards if they wanted to become lawful permanent residents and then citizens. + Many of the presidential candidates reacted cautiously to the agreement. + Senator Barack Obama, Democrat of Illinois, said he ''did not want to prejudge the deal'' before he had an opportunity to study the legislation. Mr. Obama said that he favored strengthening border security and creating a pathway to citizenship, but that he was troubled by the temporary-worker system and the proposed point system. + ''Those two things represent significant changes,'' Mr. Obama said. ''Whether they work to stabilize the immigration system and whether they are just and human is something that I'm still concerned about.'' + Senator Hillary Rodham Clinton, Democrat of New York, said she had not had an opportunity to review the proposal. + Mrs. Clinton said she would examine the proposal ''to see if it honors our nation's principles and proud immigrant heritage while also respecting the rule of law.'' + John Edwards, another Democratic candidate, said he had concerns about parts of the proposal, including a ''poorly conceived guest worker program.'' + Among the major Republican contenders, immigration has been a potentially troubling issue for Mr. McCain and Rudolph W. Giuliani because of their previous stances. + Another Republican candidate, Mitt Romney, the former Massachusetts governor, issued a blistering denunciation of the proposal, saying it was ''unfair to the millions of people who have applied to legally immigrate to the U.S.'' \ No newline at end of file diff --git a/nyt-2007_small/1848320.xml.txt b/nyt-2007_small/1848320.xml.txt new file mode 100644 index 0000000..4f5d391 --- /dev/null +++ b/nyt-2007_small/1848320.xml.txt @@ -0,0 +1,27 @@ +'08 Candidates Weighing Consequences as They Take Sides on Immigration Plan + + ’08 Candidates Weighing Consequences as They Take Sides on Immigration Plan + +The bipartisan immigration proposal being taken up by Congress is putting pressure on the leading presidential candidates to take a position on the issue, which could set them up for confrontations with influential constituencies within the two parties. + After the announcement of the bipartisan plan on Thursday, Senator Hillary Rodham Clinton, the New York Democrat, and Rudolph W. Giuliani, the former New York mayor who is one of the Republican frontrunners, were initially noncommittal. Both suggested on Friday that they were open to supporting it, but only with major revisions to some of its main components. + Reflecting the complexity of the issue and the political caution surrounding it, neither of them matched the embrace of the legislation on Thursday by Senator John McCain, the Arizona Republican. + Mr. McCain already faces a direct clash with another of the Republican candidates, Mitt Romney of Massachusetts, who has come out against the bill as he intensifies his efforts to win the support of conservatives who are wary of Mr. McCain and Mr. Giuliani. Mr. Romney's opponents said his position had shifted from more moderate views that he voiced a few years ago. + On Friday, the Romney campaign unveiled a television commercial about illegal immigration that it said would run starting this weekend in New Hampshire and Iowa. + Mr. McCain's position was also assailed on Friday by conservative commentators, who object in particular to the provisions of the legislation that could ultimately grant legal status to many of the estimated 12 million immigrants who are in the United States illegally. + Mr. Giuliani's emphasis has been on whether the legislation would adequately protect the nation from terrorists who might enter the United States illegally. + In an appearance on Friday in Orlando, Fla., he supported the idea of compromise as long as it included a system for registering the people who are currently in America illegally and issuing them identification cards. + ''I think this idea of working things out between the Democrats and the Republicans -- and each side has to make some compromises in order to get there -- then I can see a lot of flexibility there to get that accomplished,'' he said. + Mrs. Clinton, an aide in her Senate office said, will focus on the provision that would de-emphasize family ties in granting visas, an element of the bill that has drawn fierce objections from many groups and elected officials representing Hispanics. + Mrs. Clinton will try to limit the impact on Hispanic immigrants by offering an amendment to reunite lawful permanent residents with their spouses and minor children by exempting those family members from the visa cap in the bill, the aide said. + When Mrs. Clinton was asked about the bill in New Orleans on Friday, however, she avoided stating any precise position and instead highlighted her support for both toughening border security and providing a path to citizenship for illegal immigrants. + Democrats face a complicated nest of competing interests, including the outcry of some labor unions about the proposed guest worker program and concerns among Hispanic voters that the legislation will severely curtail the ability of families in poorer Latin American countries to settle legally in the United States. + Major labor unions are split on the issue, but there is widespread labor concern about the provision that would set up a temporary guest worker system with no chance of citizenship. + Not only could those workers take away jobs from Americans, these opponents said, but they could also become a permanent underclass. + ''The proposal unveiled today includes a massive guest worker program that would allow employers to import hundreds of thousands'' of temporary workers every year to perform permanent jobs throughout the economy, John J. Sweeney, president of the AFL-CIO, said in a statement. + John Edwards, the former senator from North Carolina whose campaign has focused on trying to rally traditional union workers with a populist message, said he was pleased to see movement on immigration reform. But, in a statement, Mr. Edwards added, ''I have some real concerns about parts of this bill, including the poorly conceived guest worker program.'' + Senator Barack Obama, Democrat of Illinois, was similarly cautious. + ''Without modifications, the proposed bill could devalue the importance of family reunification, replace the current group of undocumented immigrants with a new undocumented population consisting of guest workers who will overstay their visas, and potentially drive down wages of American workers,'' Mr. Obama said in a statement. + Democrats and Republicans must calculate how their choices will affect them politically, not just in early states like Iowa but also in states like New York, Florida, California and Texas that have large immigrant populations. + Muzaffar Chishti, the director of the Migration Policy Institute's office at the New York University law school, said that the pressure was going to quickly build for everyone to declare firm positions. + ''Next week is going to be very, very critical,'' Mr. Chishti said. ''The pressure on the senators between now and Monday is going to be intense.'' + THE IMMIGRATION LEGISLATION \ No newline at end of file diff --git a/nyt-2007_small/1850400.xml.txt b/nyt-2007_small/1850400.xml.txt new file mode 100644 index 0000000..5201926 --- /dev/null +++ b/nyt-2007_small/1850400.xml.txt @@ -0,0 +1,22 @@ +In Clinton Aide's Advice, An Early Voting Dilemma + + In Clinton Aide’s Advice, an Early Voting Dilemma + +Tucked away in a Hillary Rodham Clinton campaign memorandum that leaked out this week was a brief note about a quirk in next year's primary election calendar that could have a profound effect on the strategies and spending of all the candidates. + The memorandum, by Mike Henry, the deputy campaign manager for Mrs. Clinton, made a case for ignoring Iowa's first-in-the-nation caucuses next Jan. 14 and devoting the candidate's limited time and resources to contests in much bigger states in the following three weeks, including Florida, Arizona, California, New Jersey, Georgia and Texas. + Mr. Henry noted that all of those states, and several others whose primaries are set for early February, will be mailing out millions of absentee ballots in the weeks before Iowans gather for their caucuses, potentially diminishing Iowa's importance. Iowa, along with New Hampshire, which holds its primary a week after the Iowa caucuses, have jealously guarded their status as early arbiters of who's hot and who's not. + But next year, because of the extreme compression of the primary calendar, millions of voters will have the chance to register their choices before the Iowa and New Hampshire results are in. Mr. Henry warned that the Clinton campaign could not afford to ignore the vastly greater numbers of potential early voters in the big states that vote right after tiny Iowa and New Hampshire. + ''Iowans will not be the first to vote,'' Mr. Henry wrote in his memorandum. ''Hundreds of thousands of voters will be voting in California, Florida and Texas. We must fund an expensive paid communications and a vote by mail/early vote program in these mega-states.'' + Ms. Clinton dismissed the advice to skip Iowa, where some recent polls show she is running third behind her chief Democratic rivals, former Senator John Edwards of North Carolina and Senator Barack Obama of Illinois. But her campaign has been silent on what it plans to do to woo early voters in the other states. + This early-voting phenomenon is a big deal for all the candidates, and they are struggling with it. In California, for example, a third of the electorate in both primary and general elections votes absentee. The state will mail out some three million absentee ballots for the Feb. 5 primary starting on Jan. 7 (overseas and military voters from California will get their ballots in mid-December). Each presidential campaign will have to decide whether it is worth the money to try to reach these voters by mail, radio, television or door-to-door canvassing, a huge expense in a state the size of California. + The same is true for Florida, which will hold its primary on Jan. 29 and mail its absentee ballots more than two weeks earlier. Mr. Henry noted that tens of thousands of voters in Arkansas, Arizona, New Jersey, New Mexico, Oklahoma, Tennessee, Utah, Colorado, Georgia, North Carolina and Texas will receive their ballots before Iowans crunch through the snow to their caucuses. Most of these states offer so-called no-excuse absentee voting, meaning that any registered voter can request an absentee ballot without giving a reason. + New York, which will also hold its primary on Feb. 5, provides absentee ballots only to those who certify they will be away on Election Day or are disabled. + ''Candidates are facing this much more complicated environment and are going to have to get out on the ground in these other states much earlier,'' said Paul Gronke, a political science professor at the University of Oregon and an expert on early voting. ''It's a fascinating study in voter behavior, too. Will voters in California hold their ballots, waiting to see what Iowa and New Hampshire do? Or will they feel they already know these candidates well enough by then, especially Hillary and Obama?'' + Mr. Gronke said he would be surprised if more than 10 percent of Florida or California voters cast their ballots early, and those voters very likely were strongly committed to their candidate and would not be swayed by the results in Iowa. In any case, their votes would not be counted until some weeks after the Iowa and New Hampshire results were announced. + ''Reporters -- and the public -- rely on results, not on votes that may have been cast but have not yet been counted,'' he said. ''Certainly, it changes the political landscape when some percentage of the voters cast their ballot prior to Iowa. But I don't see any reason that this changes the importance of Iowa.'' + Carl Forti, a spokesman for Mitt Romney, the former governor of Massachusetts and one of the leading Republican presidential contenders, said that this foreshortened election cycle is unlike any other, and the potentially large numbers of early voters make it that much tougher. ''You have to find a new way to navigate the field,'' Mr. Forti said. He said that the Romney campaign had not yet decided whether, where or how to focus on early voters. But, he added, ''It is something you have to factor into your planning.'' + A senior strategist for Mr. Edwards, who has said he believes he must win in Iowa or New Hampshire to remain viable, said that the campaign would devote few resources to absentee voters in other states. ''We don't think many people will vote before Iowa,'' the Edwards aide said. ''If you see us spending money outside the first four states, it's because we're rolling in dough. But there's no way it's worth risking taking money and great staff out of Iowa and New Hampshire.'' + Aides to other candidates declined to discuss the question, saying it was a matter of internal strategy that they did not want to share with their competitors. But an aide to Senator John McCain, Republican of Arizona, said facetiously, ''I know I'm looking forward to casting my ballot on Christmas Day.'' + But he added, ''I'd be negligent in my job if I didn't consider the amount of early voting that is going to take place. I'd also be negligent if I told you how we intend to exploit it.'' + POLITICAL MEMO + Correction: May 30, 2007, Wednesday An article on Sunday about early and absentee voting in next year's presidential primaries misidentified the academic affiliation of Paul Gronke, a professor of political science who is an expert on early voting. He is at Reed College in Portland, Ore., not the University of Oregon. \ No newline at end of file diff --git a/nyt-2007_small/1850573.xml.txt b/nyt-2007_small/1850573.xml.txt new file mode 100644 index 0000000..fe8e4f1 --- /dev/null +++ b/nyt-2007_small/1850573.xml.txt @@ -0,0 +1,32 @@ +WEALTHY ENCLAVE OFFERS WINDFALL FOR CANDIDATES + + Wealthy Enclave Offers Windfall for Candidates + +Senator John McCain made his pitch to this gilded shoreline suburb back in April. Former Gov. Mitt Romney of Massachusetts came on May 7, followed one night later by former President Bill Clinton on behalf of his wife, Senator Hillary Rodham Clinton. Last weekend, it was back-to-back appearances by Senator Barack Obama, topped off on Sunday with a visit from Rudolph W. Giuliani, the former New York mayor. + With the mansions along its winding back roads now awash in hedge fund money, Greenwich has joined New York, Los Angeles and Silicon Valley as must stops on the presidential fund-raising tour, with prominent locals now boasting of candidate scuff marks on their basketball courts, Secret Service T-shirts in their closets and framed pictures of their children with the candidates on their mantels. For a town that has wealth and corporate clout to spare, the fund-raisers fill a void: access to a potential White House resident. + ''The people I know are absolutely thrilled to have presidential candidates come in because, as you know, Connecticut is somewhat irrelevant in the primary process,'' said Adam Wood, a longtime Democratic organizer in Connecticut. ''So it's a great opportunity for Connecticut and for Greenwich to have the future president of the United States there.'' + As Connecticut's richest town, Greenwich has always held some allure for politicians looking to raise money and make influential friends, but with several of the current presidential contenders wanting to raise upwards of $500 million to stay competitive through the general election, the richest of the town's 61,000 residents are being wooed more than ever. + An analysis by The New York Times of campaign filings through March 31 tracked $1.038 million in gifts related to the presidential race from donors with Greenwich ZIP codes. That windfall is roughly two-fifths the $2.56 million kicked in by those same ZIP codes during the entire 2004 presidential race. The preliminary 2007 tally does not capture more than $1 million believed to have been raised since then, as the pace of fund-raising here has intensified. + In one notable break with the past, Greenwich money is increasingly going to Democrats, a reflection of national trends. Of the $1.038 million donated through March, six Democratic candidates shared 54 percent of the pot, and three Republicans claimed 46 percent. Democrats got 45 percent of the $2.56 million raised the last time around, and the incumbent Republican candidate, President Bush, whose father was born here, swept the other 55 percent. + People who are active in politics offer several explanations. With Greenwich's hedge fund industry booming, many of its newest moguls do not have entrenched political allegiances and reflect the larger shift toward Democratic candidates seen elsewhere in Fairfield County. Last year's Senate bid by Ned Lamont, a local Democrat, may also have left the town more fluid than it once was, as some Republicans registered as Democrats for the primary and then switched back. + With so many events to choose from, some well-to-do residents are attending more than one, taking comfort in the fact that donations are generally capped by law at $2,300 for primaries and $2,300 for the general election. ''Some people just like to see the candidate and don't mind buying the ticket,'' said James Lash, the town's first selectman. ''It's like going to the theater.'' + For those who can, the protocol becomes clear. Eat first because campaigns can be stingier with the treats than hosts would be if they were paying for the spread. Allison Frantz, for instance, acknowledged that holding the Romney event in her home was ''a challenge because it was a really small budget'' but said she found ways to be creative, like using cookies in lieu of floral centerpieces. ''They're not going for the food,'' her friend, Elizabeth Oberbeck, said of the crowd. + Do not expect anything expensive to drink for the same reason. Ooh and aah when the hostess shows you her child's history project that the former president signed or regales you with details about how this politician or that tossed a football with her brood. + ''The money is getting younger, so all these people have young kids,'' said a frequent guest who requested anonymity because she did not want to offend her hosts. She called the mementos ''the newest trophy'' for the set that has it all. + ''It's cool and impressive to be hosting these things,'' the guest said. ''It's what you do when you can have it all and your money can buy you so many things. But to actually have a presidential candidate or former president come to your home and visit with your friends, it's a rare experience, and it's joining the things in Greenwich that are de rigueur about living in the community.'' + Harry Clark, a public policy consultant based in Greenwich who was a co-host for a McCain fund-raiser at the Belle Haven Country Club in April, said, ''Everyone on the Republican side is window-shopping, and on the Democratic side, there's tremendous curiosity about Obama.'' + A former top aide to Jack Kemp, the former quarterback and Republican politician, Mr. Clark was intrigued enough himself to have written a check that entitled him to attend one of Mr. Obama's recent ''drop-bys,'' as the Obama campaign called them. + Judging from what was reported through March, the current leaders of the pack in Greenwich are Mr. Romney, with $200,627 on the Republican side, and Senator Christopher J. Dodd of Connecticut, with $190,900 on the Democratic side. Close behind are Mrs. Clinton, a New York Democrat, with $166,550, and Mr. Giuliani, a Republican, with $154,350. Mr. McCain, an Arizona Republican, raised $124,000, and Mr. Obama, an Illinois Democrat, raised $106,000. + The campaigns were reluctant to discuss their fund-raising strategies or their recent galas, which were roped off to reporters. + The rankings are expected to shift when second-quarter results are reported. Mr. Romney's supporters say they raised at least $550,000 from the dinner he attended with 350 guests on May 7 at the waterfront home of Mrs. Frantz and her husband, L. Scott Frantz, a financier who has been a backer of Republican causes. Admission was $1,000. For $2,300, the campaign threw in a photograph with the candidate. + Organizers of a benefit the following night for Mrs. Clinton, which was held at the home of Clifton and Deborah Robbins, charged $2,300 per ticket but limited the attendance so they could bill the event as an ''intimate'' dinner. Not to be left out, Mr. Giuliani also asked $2,300 a head of the 130 or so people who turned out to meet him last Sunday at the home of Douglas Korn, a Bear Stearns executive, and Mr. Korn's wife, Elizabeth. + It was Mr. Obama's four-hour dash through town, however, that had everyone chatting, mostly about the hundreds of thousands of dollars he gleaned. The evening began at the waterfront home of the billionaire Paul Tudor Jones II, a hedge fund titan. His home, worth an estimated $25 million, features an underground garage roomy enough for more than 25 cars. It has been described in Vanity Fair as ''a cross between Tara and a national monument.'' + Another billionaire, George Soros, a longtime supporter of liberal causes, was a co-chairman of the event, which drew 300 guests who were asked to pay $2,300 apiece. A private reception was tossed in for about a dozen people who had raised $25,000 or more for the candidate. By many accounts, waiters in white tuxedos flitted among the crowd, and the supermodel Tyra Banks added sparkle to the guest list. + Leaving that affair, Mr. Obama then met with a mostly Greenwich crowd of 150 people in a gated section of town known as Conyers Farm, which straddles Connecticut and New York. His host was the former New York Knicks basketball player Allan Houston and his wife, Tamara, whose home has a Greenwich address but is in Armonk, N.Y. Tickets cost $1,000 to $2,300. + When Mr. Obama took note of his host's basketball court and seemed eager to take a few shots, Mr. Houston joked that he would not think of wiping the scuff marks of ''the future president'' off the court, Greenwich Time reported. + When estimating how much was raised by an event, campaign officials cautioned against assuming that everyone in the crowd had paid. Tom Swan, who managed the Senate campaign last year for Mr. Lamont, said people were sometimes admitted free to lend ''the impression you have this much support.'' + ''Sometimes, it's to try to cultivate donors, and sometimes there are really good volunteers or folks who had maxed out previously and you don't want to alienate them,'' Mr. Swan said. + Those who are consigned to watch from the sidelines generally take the visits in stride, said Mr. Lash, the first selectman. Doubling as the town's police commissioner, Mr. Lash must often help secure safe passage for dignitaries and deal with those who are rankled by the occasional traffic snarls. + Mr. Clinton's visit to Greenwich, which involved Secret Service protection, probably cost the town $4,000 to $5,000, Mr. Lash said -- a tenth of what it cost last fall when President Bush dropped in via helicopter at Binney Park to attend a fund-raiser for the state Republican Party, which was also held at the Frantz home. + As much as the cost, Mr. Lash recalled, the president's visit caused howls because it disrupted the girls soccer league that ordinarily uses the park. + Correction: May 30, 2007, Wednesday Because of an editing error, an article on Monday about the position of Greenwich, Conn., as a must stop on the campaign fund-raising trail misidentified the birthplace of former President George Bush. He was born in Milton, Mass. -- not Greenwich, where President Bush outraised Democrats in the 2004 campaign. \ No newline at end of file diff --git a/nyt-2007_small/1850861.xml.txt b/nyt-2007_small/1850861.xml.txt new file mode 100644 index 0000000..3b97046 --- /dev/null +++ b/nyt-2007_small/1850861.xml.txt @@ -0,0 +1,28 @@ +Obama Calls for Wider and Less Costly Health Care Coverage + + Obama Calls for Wider and Less Costly Health Care Coverage + +Senator Barack Obama proposed a major overhaul of the nation's health care system on Tuesday, aimed at covering the nearly 45 million uninsured Americans and reducing premium costs for everyone else. + In the biggest domestic policy proposal so far of his presidential campaign, Mr. Obama, the Illinois Democrat, said he would rely on a combination of the existing employer-based system and a new government program to make health insurance accessible to everyone. He also promised to reduce the cost of health insurance by helping with expenditures for catastrophic illnesses that are a major factor in driving up employers' rates. + Throwing down a challenge to a powerful industry, Mr. Obama pledged new scrutiny and new limits on the profits of the biggest insurance companies, declaring it was simply ''the right thing to do.'' + Mr. Obama would pay for his plan by not renewing President Bush's tax cuts for the most affluent Americans -- those making more than $250,000 a year -- when they expire at the end of 2010, aides said. Campaign officials estimated that the net cost of the plan to the federal government would be $50 billion to $65 billion a year, when fully phased in, and said the revenues from rolling back the tax cuts were enough to cover it. + Mr. Obama's speech came after months of criticism that his campaign lacked specificity and policy heft on complex issues like health care. It was also his first foray into an area that is widely considered the home terrain of his leading opponent for the Democratic nomination, Senator Hillary Rodham Clinton of New York, who has yet to detail her own proposal. + The Obama plan drew from a long line of Democratic policy thinking; it reflects elements of proposals put forward by the Clintons, by Senator John Kerry in his 2004 presidential campaign and by former Senator John Edwards this year. It was built on advice from several economists, including Austan Goolsbee at the University of Chicago, David Cutler at Harvard and Stuart Altman at Brandeis. + Rival Democrats, recognizing the stakes on this signature issue, responded quickly, arguing that because Mr. Obama would not require every American to have insurance, it is not a true universal health plan. + Neera Tanden, policy director for Mrs. Clinton -- who had earlier delivered an economic policy speech to compete with Mr. Obama's -- welcomed Mr. Obama to the health care debate. Ms. Tanden added, ''Senator Clinton believes that in addition to making healthcare more accessible, we have to achieve true universal health care so that every American has health care coverage.'' + Mark Kornblau, a spokesman for Mr. Edwards's campaign, said, ''Any plan that does not cover all Americans is simply inadequate.'' + Others said the Democratic plans and approaches showed more agreement than ideological division. + ''Voters will see more similarities than differences in the Democratic candidates' plans, and I think the edge will go to the candidate who convinces voters they can forge a consensus in Congress and get things done,'' said Drew Altman, president of the Kaiser Family Foundation, a health research group. + Stuart Altman of Brandeis, who served in the Nixon administration and has advised many candidates and politicians since, said the Obama plan had major similarities not just to other Democratic proposals, but also to the Massachusetts health plan passed under Mitt Romney, now a candidate for the Republican presidential nomination, when he was governor. Both are efforts to build on the existing system, with a mix of employer-based coverage and expanded public programs. But Mr. Altman said there were also differences, notably that the Obama plan puts more responsibilities on employers and focuses more on controlling costs, and avoids a requirement in the Romney plan that individuals buy coverage. + In his speech, Mr. Obama acknowledged that similar efforts to expand coverage had failed in the past, ''crushed under the weight of Washington politics and drug and insurance industry lobbying.'' + But he added, ''This cannot be one of those years.'' + His proposal includes a new requirement that employers either provide coverage to their employees or pay the government a set percentage of their payroll to provide it. Similar requirements have proven intensely controversial, notably in 1993-94, when the Clinton health care plan went down in large measure because of a small-business backlash. + Obama advisers said the smallest businesses would be exempt from this requirement. + Mr. Obama would create a public plan for individuals who cannot obtain group coverage through their employers or the existing government programs, like Medicaid or the State Children's Health Insurance Program. Children would be required to have health insurance. Subsidies would be available for those who need help with the cost of coverage. + He would also create a National Health Insurance Exchange, a regulated marketplace of competing private health plans intended to give individuals other, more affordable options for coverage. The public plan would compete in that Insurance Exchange, advisers said. + The Obama campaign revives an idea advanced by the Kerry campaign -- using the federal government to cushion employers from sudden, disastrous spikes in health expenditures, by reimbursing health plans for the cost of catastrophic illnesses among their employees. Because of this and other cost-containment proposals, Mr. Obama estimated his plan would save the typical family up to $2,500 a year in their health insurance costs. + Mr. Obama described his plan to an audience at the University of Iowa, invoking the memory of Medicare and arguing that the time had come to finish the work of Harry S. Truman and Lyndon B. Johnson. He acknowledged that it would not be easy, but had to be done, because families who played by the rules were being driven into bankruptcy because of health care costs. + ''It is simply not right that the skyrocketing profits of the drug and insurance industries are paid for by the skyrocketing premiums that come from the pockets of the American people,'' he said. ''This is not who we are. And this is not who we have to be.'' + In New Hampshire, Mrs. Clinton gave what her campaign had billed as a major policy address on income inequality. She promised to take steps to combat ''rising inequality and rising pessimism in our work force.'' + Mrs. Clinton said her prescriptions included reducing tax breaks for corporations, allowing Medicare to negotiate lower drug prices with pharmaceutical companies, eliminating incentives for companies to move jobs overseas and giving more students a chance to attend college. + ''While productivity and corporate profits are up, the fruits of that success just haven't reached many of our families,'' she said. ''It's like trickle-down economics without the trickle.'' \ No newline at end of file diff --git a/nyt-2007_small/1852320.xml.txt b/nyt-2007_small/1852320.xml.txt new file mode 100644 index 0000000..a2feecf --- /dev/null +++ b/nyt-2007_small/1852320.xml.txt @@ -0,0 +1,34 @@ +Edwards, Clinton and Obama Describe Journeys of Faith + + Edwards, Clinton and Obama Describe Journeys of Faith + +One presidential hopeful described how prayer helped him survive his son's death and his wife's cancer diagnosis. Another spoke about the Israeli-Palestinian conflict and the role of faith in forgiving those who treat others unjustly. A third said of her husband's infidelity, ''I'm not sure I would have gotten through it without my faith.'' + Intimate discussions of politics and religion have long been the province of Republican candidates for public office. But on Monday night, the three leading Democratic presidential hopefuls -- former Senator John Edwards and Senators Barack Obama and Hillary Rodham Clinton -- opened up at an unusual televised forum about their faiths, the role of prayer in their public and private lives and the ways that religion informs their views on policy and government. + Each is aiming to make historic inroads among evangelical Christians and other committed churchgoers who have up to now been most linked with the Republican base. The candidates appeared eager not just to discuss their policies but also to discuss their personal faith journeys as they spoke, one after another, at George Washington University. + The event was organized by a liberal evangelical group, Sojourners, and televised by CNN. + The participants sought to walk a fine line between appealing to religious voters, while not turning off secular voters, who represent a crucial constituency for them. + Mr. Edwards, who has spoken extensively about poverty in moral terms but has shared little about his faith, demonstrated dexterity with speaking the language of Christian belief. + When asked whether he would be willing to discuss the ''biggest sin you've ever committed,'' Mr. Edwards laughed, paused for a moment and said that the ''list is too long.'' + ''I'd have a very hard time telling you one thing, one specific sin,'' he said, drawing applause. ''If I've had a day in my 54 years that I haven't sinned multiple times I'd be amazed. We all fall short, which is why we have to ask for forgiveness from the Lord.'' + Mr. Edwards recalled growing up in the Southern Baptist Church but admitted that he had strayed as an adult. His faith, however, came ''roaring back'' in the midst of family crises. First was his son Wade's death in a car accident, and then came the diagnosis for his wife, Elizabeth, and the recent recurrence of cancer. + ''I've been through a faith journey in my life,'' he said, adding that prayer ''played a huge role in my survival'' in those difficult moments. + ''It's the Lord who got me through,'' he said. + Mrs. Clinton, who appeared comfortable chatting on stage with the CNN interviewer, Soledad O'Brien, and later roaming the stage while addressing the audience, drew the most rounds of applause. There were moments on stage that had an almost confessional quality for her. + Ms. O'Brien noted that Mrs. Clinton had shared relatively little about her faith in public but then carefully broached what has largely been the one issue that has been largely off limits in Mrs. Clinton's campaign, her husband's infidelity, asking whether her faith helped her deal with it. + ''I'm not sure I would have gotten through it without my faith,'' Mrs. Clinton said. + Mrs. Clinton said she took her faith ''very seriously and very personally'' but went on to say she came from a faith tradition, Methodism, that is ''perhaps a little too suspicious of people who wear their faiths on their sleeves.'' + She admitted that talking about her faith in public ''doesn't come naturally to me,'' saying she often flashed back to ''the Pharisees and all of the Sunday school lessons and readings I had as a child.'' + She expressed gratitude for close friends and others who she said were praying for her, describing them as ''prayer warriors'' who ''sustained me through a very difficult time.'' + ''I am very grateful I had a grounding in faith that gave me the courage and the strength to do what I thought was right, regardless of what the world thought,'' she said, drawing a rousing round of appreciative applause. + When Ms. O'Brien asked what she asked God for in her prayers, Mrs. Clinton drew laughter from the audience when she said, ''Sometimes, I say, 'Oh Lord, why can't you help me lose weight.' '' + Mr. Obama dwelled somewhat more on policy and global concerns than on his personal faith or Scripture, in large part because of the nature of the questions that he faced. But he also found ways to interlace religion and policy. + Asked whether he believed that God took sides in a war, Mr. Obama reached for a famous quotation of Lincoln, about asking whether the nation is on God's side. + At the same time, he said, it was important to remain ''our brother's keeper, our sister's keeper'' to advance the causes of justice and freedom. + Mr. Obama said he believed that evil existed in the world, noting, 'I do think when planes crash into building and kill innocents, there's evil there.'' In other times of violence and war, however, he saw just causes like the Civil War and the defeat of fascism and the liberation of Europe. + He also said that his ''starting point as president is to restore that sense that we are in this together'' and that this commitment rose out of his faith. He promised to build alliances across partisan lines to improve early childhood education, children's nutrition, workers' pay and efforts to put criminal offenders on a better path. + ''The notion that we take away education programs in prisons, to be tough on crime, makes absolutely no sense,'' Mr. Obama said. + The event was the first in recent memory by Democrats that focused explicitly on faith and its values. It highlights how far the party has come since the 2004 presidential election in its efforts to appeal to religious voters and the openings Democrats see if the Republicans nominate a candidate who supports abortion rights and gay rights like Rudolph W. Giuliani or one who would be the first Mormon president, Mitt Romney. + Mara Vanderslice, director of religious outreach for the Kerry-Edwards campaign in 2004, said it would have been almost unimaginable for Democratic candidates to have participated in such an event in 2004. + Ms. Vanderslice recalled how difficult it was to nudge Mr. Kerry to talk about his Roman Catholic faith in a substantive way during the campaign. + ''We would never have seen something like this last cycle,'' she said. + THE 2008 CAMPAIGN \ No newline at end of file diff --git a/nyt-2007_small/1852322.xml.txt b/nyt-2007_small/1852322.xml.txt new file mode 100644 index 0000000..d2897ea --- /dev/null +++ b/nyt-2007_small/1852322.xml.txt @@ -0,0 +1,11 @@ +Trying to Stay Above the Fray + +A few months ago, Senator Hillary Rodham Clinton said she would ''deck'' anyone who attacked her during the presidential campaign. The idea was that she would fight back rather than allow herself to be ''Swift boated'' the way Senator John Kerry was in 2004, when he found his Vietnam War record assailed. + Sure enough, both the Republican and Democratic presidential candidates have been stepping up their verbal attacks on Mrs. Clinton. So far she has stayed above the fray. It is a classic front-runner strategy. But she is also trying not to fuel critics who say she is a polarizing figure. + Rudolph W. Giuliani criticized her during a Republican debate in mid-May, portraying her as opposing the free-market system and favoring high taxes. + Mrs. Clinton's response? ''I didn't see the debate.'' + Last week, Mitt Romney said that as president, Mrs. Clinton would create a socialist-style welfare state. Her platform, he told voters in Iowa, ''wouldn't even get her elected in France.'' + The Associated Press reported that the Clinton campaign declined to respond to Mr. Romney. + In Sunday night's Democratic debate, former Senator John Edwards accused Mrs. Clinton of being a follower, not a leader, for not speaking out on the Iraq war financing bill. Mrs. Clinton's response: ''The differences among us are minor,'' she said. ''The differences between us and the Republicans are major.'' + KATHARINE Q. SEELYE + THE CAUCUS: RACE \ No newline at end of file diff --git a/nyt-2007_small/1852323.xml.txt b/nyt-2007_small/1852323.xml.txt new file mode 100644 index 0000000..089e2a7 --- /dev/null +++ b/nyt-2007_small/1852323.xml.txt @@ -0,0 +1,10 @@ +Your Turn, G.O.P. + +Tonight it will be the Republicans' turn to debate in New Hampshire, and two issues are likely to figure prominently: immigration and terrorism threats, particularly the reported plot to blow up fuel lines at Kennedy International Airport. + Rudolph W. Giuliani, the former mayor of New York, has already incorporated the plot into his stump speech. On Saturday, after three people were arrested in connection with it, he told reporters in Florida that it served as a reminder that ''we're at war.'' + Mr. Giuliani has sometimes been accused of fear-mongering. But the airport example flows seamlessly into his message, which is that evildoers are out to kill vast numbers of Americans, as they did on Sept. 11, and that he would keep the country safe. + Certainly it reinforces his argument against the Democrats, particularly former Senator John Edwards, who reiterated at the Democratic debate on Sunday that the phrase ''war on terror'' was a political slogan used by President Bush to justify questionable actions, not a real plan. + But what will Mr. Giuliani make of Senator Hillary Rodham Clinton's comments? She did not agree with Mr. Edwards, adding, ''I believe we are safer than we were'' before Sept. 11, an off-script sentiment for Democrats. + The 7 p.m. debate, to be broadcast on CNN, will give Senator John McCain of Arizona and Mitt Romney, the former governor of Massachusetts, the opportunity to confront each other on the immigration bill after a week of veiled swipes. + KATHARINE Q. SEELYE + THE CAUCUS \ No newline at end of file diff --git a/nyt-2007_small/1852583.xml.txt b/nyt-2007_small/1852583.xml.txt new file mode 100644 index 0000000..0c6f4bd --- /dev/null +++ b/nyt-2007_small/1852583.xml.txt @@ -0,0 +1,25 @@ +G.O.P. Candidates at Odds Over Immigration Overhaul + +The Republican presidential candidates waged verbal combat last night in the Congressional battle over a proposed immigration overhaul, with Senator John McCain, an author of a pending compromise plan, defending the controversial proposals in a Republican debate in Goffstown, N.H., and demanding that his competitors offer ''a better idea.'' + Mr. McCain found himself at odds with former Mayor Rudolph W. Giuliani of New York and former Gov. Mitt Romney of Massachusetts, who each criticized the Senate plan as woefully lacking in border security and immigration enforcement measures, including a uniform method to identify immigrants and limits on certain types of visas. + ''It's a typical Washington mess,'' Mr. Giuliani said in his first heated face-off of the campaign with his past political ally, Mr. McCain. ''Everybody compromises, four or five compromises, and the compromises leave you with the following conclusion. The litmus test you should have for legislation is, is it going to make things better? And when you look at these compromises, it is quite possible it will make things worse.'' + Mr. Romney, who has been accused by Mr. McCain of ''pandering'' to voters by criticizing the immigration proposal, passed up an invitation by the moderator of the debate, Wolf Blitzer of CNN, to hit back. + ''Well, he's my friend,'' Mr. Romney said of Mr. McCain, downplaying their recent feud. ''He campaigned for me two times.'' Then Mr. Romney pivoted to criticize the plan's ''Z visas'' as a hole that would allow many illegal immigrants to stay in the United States ''for the rest of their lives.'' Under the Z-visa provision, if immigrants paid fines and passed background checks, they could obtain a visa that would be valid for four years and could be repeatedly renewed. + But Mr. Romney was not completely correct in saying illegal immigrants would be put ahead of all people who have been waiting to come to this country legally. If they wanted permanent-residence visas, or green cards, they would have to wait in line behind people who have already applied. + Mr. McCain forcefully defended the plan, citing its employment-verification system and its strict citizenship rules for illegal immigrants. And he pointedly took on Mr. Giuliani. ''Rudy, you just described our legislation, so I'd be glad to have a further conversation with you,'' Mr. McCain said. + After months of introducing themselves to voters, the leading candidates came into their third televised debate, at St. Anselm Collge, with increasingly well-formed political identities, though polls suggest that none yet have captivated wide swaths of the Republican electorate. + Mr. Giuliani again portrayed himself as the tough-minded, blunt-speaking protector of a nation in danger, saying it would have been unthinkable for the United States not to have invaded Iraq to remove Saddam Hussein from power. Mr. McCain again showed himself willing to buck party orthodoxy, not just on immigration but, to some degree, on whether English should be the national language (he noted that Navajos in Arizona have a proud culture and use their own language). And Mr. Romney again portrayed himself as a tax-cutter and a conservative, despite his moderate positions on social issues in the past. + Not on the stage here but creating his own mighty buzz because of his relatively clear conservative positions, was a Republican preparing to enter the race, former Senator Fred D. Thompson of Tennessee. Mr. Thompson appeared on Fox News immediately after the debate, and was asked about criticism he is already facing as a potential candidate. + ''It's a badge of honor to get attacked by some of these bozos,'' he said. + More so than in previous appearances, the candidates were willing to criticize President Bush and his administration. Representative Duncan Hunter of California ripped the immigration legislation, calling it the ''the Bush-McCain-Kennedy'' plan; Representative Tom Tancredo of Colorado said Mr. Bush would ''never darken the doorstep of the White House'' if Mr. Tancredo were elected president because of the candidate's anger over White House immigration and education policies and the Medicare prescription drug benefit. + But the Republicans appeared downright eager to focus their collective attention on the Democrats and in particular Senator Hillary Rodham Clinton. Former Gov. Mike Huckabee of Arkansas even introduced himself by tweaking former President Bill Clinton, saying: ''I'm from the small town of Hope. You may have heard of it. All I ask you is, give us one more chance.'' + Mr. McCain, meanwhile, questioned Mrs. Clinton as a potential commander in chief by challenging her statement, in the Democratic debate Sunday night, that the Iraq war was the responsibility of the Bush administration. + ''When Senator Clinton says this is Mr. Bush's war, that this is President Bush's war -- when President Clinton was in power, I didn't say that Bosnia, our intervention there was President Clinton's war,'' he said. ''When we intervened in Kosovo, I didn't say it was President Clinton's war.'' + At another point, Mr. McCain and another candidate, Senator Sam Brownback of Kansas, acknowledged that they did not read the classified national intelligence estimate, which was made available to all senators, before they voted to go to war. Mr. Hunter, who serves on the House Armed Services Committee, said he did read the report. + The Republican candidates were also asked if they would support pardoning I. Lewis Libby Jr. and while none of the top candidates said, outright, that they would take such a course if president, they also voiced dismay at his sentence of 30 months in prison. Mr. Giuliani was the most outspoken of the lead candidates, calling the sentence ''way out of line.'' Citing his experience both as a prosecutor and recommending pardons while working in the Justice Department under President Ronald Reagan, he said he would seriously consider it, especially in light of the harsh sentence. + When Mr. Blitzer tried to cut him off, Mr. Giuliani stopped him, saying, ''A man's life is at stake.'' He said that it was an ''incomprehensible'' situation made all the more disturbing because there was no underlying crime, and that he had a real problem with the sentence. + Mr. Romney, following Mr. Giuliani, also criticized the sentence and left the door open for a pardon. Mr. McCain, who answered first, said that because the case was being appealed he did not think it proper to comment. + On climate change, a matter that divides many Republican voters, Mr. Giuliani said that Americans had to ''accept the view'' of scientists that global warming is real and caused by human activity. He said it was ''frustrating and really dangerous for us to see money going to our enemies'' in the form of payments for foreign oil. + Several candidates were also asked about ways they, in office, would put President Bush to work as a former president. Former Gov. Tommy Thompson of Wisconsin quipped, ''I certainly would not send him to the United Nations,'' which stirred some slightly awkward laughter from the audience and blank stares from his rivals. (The other participants in the debate were former Gov. Jim Gilmore of Virginia and Representative Ron Paul of Texas.) + Mr. Giuliani also injected a moment of humor into the debate when he was asked about the writings of a Catholic bishop who was critical of Mr. Giuliani's support of abortion rights. As Mr. Giuliani began to defend himself, there were crackles of lightning outside the debate hall -- prompting other candidates to step away from Mr. Giuliani, as if a bolt were about to strike him. + ''For someone who went to parochial schools all of his life,'' he said, ''this is a very frightening thing that is happening right now.'' \ No newline at end of file diff --git a/nyt-2007_small/1853349.xml.txt b/nyt-2007_small/1853349.xml.txt new file mode 100644 index 0000000..c425706 --- /dev/null +++ b/nyt-2007_small/1853349.xml.txt @@ -0,0 +1,38 @@ +Hispanic Voters Gain New Clout With Democrats + + Hispanic Voters Gain New Clout With Democrats + +Helped by the fight over immigration, Democratic presidential candidates are courting Hispanic voters like never before, prompted by a string of early primaries in states with sizable Hispanic voting blocs. + It has forced candidates to hire outreach consultants, to start Spanish-language Web sites and to campaign vigorously before Hispanic audiences. + The battle for Hispanic voters is a result of the decision by several states with large Hispanic populations to move their presidential primaries to early 2008, including California, Florida and New York. Roughly two-thirds of the nation's Hispanic residents live in nine of the states holding Democratic primaries or caucuses on or before Feb. 5. + Republican and Democratic strategists, as well as independent analysts, say the influence of Hispanic voters is likely to be amplified next year because of an unusually intense response in many Hispanic communities to immigration policy. Conservative Republicans, with the help of some left-leaning Democrats, teamed up on Thursday to derail an immigration bill in the Senate that would have provided a path to citizenship for illegal immigrants. + It is in the new early primary states where Democrats hope the outreach efforts bear fruit. In the last presidential election, Hispanic voters accounted for a significant part of the overall Democratic primary electorate in California (16 percent), New York (11 percent), Arizona (17 percent) and Florida (9 percent), all states that will hold primaries by Feb 5. + Sergio Bendixen, a pollster hired by Senator Hillary Rodham Clinton's campaign to study Hispanic voting trends, said: ''The Hispanic vote has never been all that important in the presidential primary process in the United States. But that will change in 2008.'' + At this early stage, Mrs. Clinton, a New York Democrat, appears best poised to benefit from the heightened Hispanic role in the primary process. She has already captured a prized endorsement, of Mayor Antonio R. Villaraigosa of Los Angeles, one of the nation's most prominent Hispanic politicians. + Mrs. Clinton is also well known and liked by many Hispanics, with several national New York Times/CBS News polls from the past few months showing that about 60 percent of registered Hispanic voters who identify themselves as Democrats have a favorable view of her, while a quarter do not. + Meanwhile, Senator Barack Obama, Democrat of Illinois, remains a blank slate to many Hispanic voters, polls show, with 40 percent having no opinion of him. But his aspirational biography could prove a draw as more Hispanic voters get to know him. + Former Senator John Edwards is even less well known among Democratic Hispanic voters. While a third have a positive view of Mr. Edwards and fewer than 10 percent have an unfavorable view of him, 6 in 10 are unable to offer an opinion. + The only Hispanic in the race, Gov. Bill Richardson of New Mexico, a Democrat, is working to build a base and establish a political identity beyond the Southwest. + Many Democrats were as troubled by the Senate immigration bill as were Republicans, but for decidedly different reasons. Mrs. Clinton expressed concerns about the legislation, particularly a provision that makes it harder for legal immigrants in the United States to bring relatives from abroad. Mr. Obama said that he would have supported the bill, but that he too had similar concerns about the provision, according to his aides. + On the Republican side, two of the main candidates, Rudolph W. Giuliani and Mitt Romney, opposed the immigration bill, while Senator John McCain played a main role in drafting the legislation, only to face a huge backlash from conservative Republicans raising alarms about what they call a flood of immigrants. + The bill's setback -- a major defeat for President Bush -- could complicate Republican efforts to win over the fast-growing Hispanic electorate and help Democrats solidify their hold on these voters, an electoral prize expected to increase in importance in coming decades. Surveys showed that Hispanics were a small part of the Republican primary vote in 2000, with their greatest influence being in California, where they made up 9 percent of the vote. + The debate over immigration has spurred Hispanic leaders and voters to mobilize like few issues in recent memory have. The National Association of Latino Elected Officials has joined with the Hispanic television network Univision on a national campaign to help Hispanic residents fill out citizenship applications and to help those who are already citizens register to vote. + Stephanie Pillersdorf, a spokeswoman for Univision, said the number of Hispanic residents who had applied for citizenship in Los Angeles County alone had gone up 146 percent since the campaign started several months ago. + The scramble for Hispanic support is evident both within the campaigns and out on the trail. + On Friday, Mrs. Clinton spoke to Hispanic leaders in the Bronx, where she accused Republicans of undermining the immigration bill in the Senate. ''The bill was mostly killed by people who don't want any immigration reform and don't want a path toward legalization,'' she said. ''There's a very big anti-immigrant feeling that is influencing the problem right now, particularly on the Republican side.'' + Earlier this month, Mr. Obama traveled to Nevada, a heavily Hispanic state that moved its caucus to Jan. 19, and sat down for interviews with Spanish-language television and newspaper reporters. + Mr. Edwards, who hopes his populist appeal will draw support from Hispanics, is dispatching his political director, David Medina, to meet with members of Democratic Hispanic Caucus of Florida. Mr. Richardson alternates between English and Spanish on the campaign trail. Senator Christopher J. Dodd, Democrat of Connecticut, also often likes to display his fluency in Spanish, including when he announced his candidacy on CNN en Español. + Republicans have been making similar efforts. Mr. McCain has been making appearances before Hispanic audiences around the country, including in Miami, where he recently gave a speech on immigration. He also has access to a deep bench of prominent Hispanic leaders who fill in for him on Spanish-language radio and television programs, including Representatives Lincoln Diaz-Balart, Mario Diaz-Balart and Ileana Ros-Lehtinen, all of Florida. The senator himself has also made appearances on Univision and Telemundo. Mr. Romney, in turn, announced on Friday the creation of a steering committee to help him attract Hispanic voters. + Strategists for several Democratic campaigns say the new calendar has set the stage for Hispanic voters to have much more influence in picking the parties' presidential nominees than they did when states like Iowa and New Hampshire were essentially alone among the early states in the nominating process. + In fact, in the 2004 race, Senator John Kerry did not assemble a Hispanic outreach and media operation until about five months before the general election. + By contrast, the Clinton campaign has already put in place a driven Hispanic outreach team that, among other things, issues press releases in Spanish on a regular basis and has a stable of Spanish-speaking surrogates to fill in for Mrs. Clinton at events that focus on Hispanics. It has also assigned a prominent role to its campaign manager, Patti Solis Doyle, a woman of Mexican descent who has been one of Mrs. Clinton's most trusted advisers and friends since her days as first lady of Arkansas. Mrs. Doyle, who played a crucial role in getting the recent endorsement from Mr. Villaraigosa, has made herself available for interviews with Hispanic organizations of all sorts. + Democrats are optimistic about their prospects of making large gains among Hispanic voters, mindful of the progress they made in the 2006 midterm elections, when only 26 percent of Hispanics voted for Republican Congressional candidates. That was down from 44 percent in 2004, when Mr. Bush was at the top of the ticket, according to nationwide exit polls conducted by Edison/Mitofsky. + While Mr. Bush's popularity with Hispanics had been a factor in drawing large numbers of them to the Republican Party, many Hispanics appear to be returning to the Democratic fold as conservative efforts gained momentum last year to restrict immigration and build a wall along the Mexican border. + Democrats are doing what they can to encourage that return. Mr. Obama has traveled to Nevada several times to meet with members and leaders of a culinary workers' union, most of whom are Hispanic women who work in Las Vegas hotels and casinos. The Obama campaign says the union could play a decisive role in generating voter turnout when the state holds its caucus next January. + The campaign is also sending dozens of volunteers this weekend to pass out Spanish-language literature in heavily Hispanic cities like Los Angeles, San Diego, Las Vegas, Phoenix, Houston and San Antonio, and is making videos available on its Web site with closed captioning in Spanish. + Mr. Edwards, in turn, is betting that his antipoverty campaign of the last few years, including helping unions organize in industries with large numbers of Hispanic workers, will give him an edge. + Earlier this year, he met with Arturo Rodriguez, the president of the farm workers' union, and several hundred union members in Fresno, Calif. Mr. Edwards's campaign has also sent prominent Hispanic supporters to act as surrogates for him on the campaign trial, including Patricia Madrid, the former attorney general of New Mexico, who recently went to Nevada to meet with Hispanic politicians and activists. + If any candidate can appeal to the ethnic pride of Hispanic voters, it is Mr. Richardson, the New Mexico governor, who often points to his Mexican roots (his mother is a native of Mexico) when appearing before Hispanic audiences. + The main problem for Mr. Richardson is that he is a relatively unknown figure among Hispanic voters, as well as the general electorate. To raise his profile among Hispanics, Mr. Richardson has turned to prominent Hispanics, including Gloria Molina, a Los Angeles County supervisor, who introduced him at the rally where he recently announced his candidacy. + David Contarino, Mr. Richardson's campaign manager, predicted that his candidacy would become a matter of ''interest and pride'' among Hispanic voters once they learned of his record and roots. + ''His name is Bill Richardson; that does not necessarily communicate his background,'' Mr. Contarino said dryly. \ No newline at end of file diff --git a/nyt-2007_small/1853955.xml.txt b/nyt-2007_small/1853955.xml.txt new file mode 100644 index 0000000..222f143 --- /dev/null +++ b/nyt-2007_small/1853955.xml.txt @@ -0,0 +1,9 @@ +Months Before First Votes, Another Kind of Counting + +The first primary votes are still more than half a year off, but the next installment of an important proxy contest -- the money primary -- is only weeks away. So the presidential candidates are all shaking the money tree this week, hoping for impressive tallies after the second quarter-reporting period ends June 30. + Several candidates are prospecting in California, including Senator Barack Obama, Mitt Romney and Senator John McCain. + John Edwards tried something a little different for his birthday Sunday. His campaign posted a video on YouTube of two of his top aides, Joe Trippi and Jonathan Prince, baking him a pecan pie and asking donors to give $6.10 to commemorate the date of his birth, $19.53 for the year he was born or $54 to match his age. The campaign said it took in a quarter of a million dollars from 10,000 donors. + Senator Hillary Rodham Clinton is expected to raise more than $2 million this week using more tried and true methods, according to a recent internal fund-raising memorandum, with events in Baltimore, Philadelphia and Washington, and in Oklahoma and Texas. Rudolph W. Giuliani has events set in Pennsylvania this week. + Money matters, and not only for what it can buy. Mr. McCain's weak first-quarter showing created the perception that his campaign was struggling; Mr. Romney and Mr. Obama got a bounce from their strong showings. Now the campaigns are trying to manage expectations for the second quarter, to put the best spin on whatever the bottom line shows. + MICHAEL COOPER + THE CAUCUS \ No newline at end of file diff --git a/nyt-2007_small/1855461.xml.txt b/nyt-2007_small/1855461.xml.txt new file mode 100644 index 0000000..6adc661 --- /dev/null +++ b/nyt-2007_small/1855461.xml.txt @@ -0,0 +1,31 @@ +By Taking On the Biggest Donors, McCain Is Taking a Big Risk, Too + + Taking On Big Donors, McCain Takes a Big Risk + +A ranking Republican on the Senate Armed Services Committee like Senator John McCain could normally bank on a bonanza of campaign contributions from the defense industry, especially if he was under pressure to raise money fast. + But as Mr. McCain races to play catch-up with his Republican presidential primary rivals before the end of the second quarter, he is only reminding military companies and lobbyists why they have never liked him. ''Defense contractors are more concerned with winning the next contract than performing on the current one,'' he charged at a recent campaign stop. + At a critical moment for him, his presidential campaign may be paying the price for a career of positions seemingly calculated to alienate constituencies that according to Washington custom should be prime sources of campaign cash. Mr. McCain's campaign filings show just $61,000 from the military industry in the first quarter -- less than half as much as the long-shot campaign of Democratic Senator Christopher J. Dodd of Connecticut, according to the nonpartisan Center for Responsive Politics. + The twist is lost on no one: a candidate who has spent decades fighting to minimize the influence of money on politics is under extraordinary pressure to scare up tens of millions of dollars to prove he can jump-start his campaign. And after months of trying to make up with factions of the conservative coalition he has snubbed in the past, fund-raising has turned into another example of the balancing act he faces as he tries to appeal to the Republican establishment without giving up his aura as a straight-talking reformer. + He does have some wealthy natural allies: American Indian casino owners, who appreciate his investigation of schemes by corrupt lobbyists to bilk tribal clients, have supported him eagerly in the past. But Mr. McCain has foresworn accepting any contributions from the casino owners, to avoid the appearance of a conflict of interest. + Now Mr. McCain is jetting to 35 fund-raising events in the 30 days of June, hoping to raise more than $100,000 at each from small groups of big donors. He is passing the hat in six states -- Georgia, Texas, Arizona, Alabama, Massachusetts and Florida -- in just the five days ending Tuesday. + Nor has he shied away from his sponsorship of the McCain-Feingold campaign finance law intended to curb the power of big donors, even as he races through his monthlong fund-raising spree. + ''There are plenty of Republicans who don't think McCain-Feingold has accomplished what it was designed to accomplish and it is infringing on free speech,'' said Lawrence E. Bathgate, a New Jersey real estate mogul and former Republican party finance director who is a top fund-raiser for Mr. McCain. ''I feel that way. But he is still my candidate.'' + His presidential campaign did receive more first-quarter money than any other candidate from telephone utilities -- $123,000, out of a total take of $12.5 million -- who fear his power as an influential former chairman of the Senate Commerce Committee. As chairman, he often pushed for rules that would help the Bell telephone companies compete against the cable companies to provide Internet access and other services. + But Mr. McCain also frequently irked the telephone industry by taking the side of consumer advocates, including trying to end the ''universal service'' fees the phone companies tack onto consumers' bills. He earned the lasting enmity of the cable companies by attacking them as price-gouging, unregulated monopolies. He was the only Republican to vote against the 1996 telecommunications overhaul, bucking virtually the entire industry on the grounds that the legislation neglected consumers. His most recent Senate campaign received less from the telephone companies than many others in Congress, including Senator Hillary Rodham Clinton, the New York Democrat. + ''He wants to be independent, to make decisions free from the influence of special interests -- God bless him,'' said Kim Bayliss, a Democratic telecommunications lobbyist. ''But those are the people with money, and they are only going to give their money because you have been with them or you are going to be with them.'' + There is of course a political benefit to Mr. McCain in being seen as untrustworthy to those who control big pools of corporate and special interest money. But there is little doubt that his record in Congress has made remaining financially competitive in the presidential race more difficult. + Once the presumptive Republican front-runner, Mr. McCain stunned political analysts with a first-quarter fund-raising total that badly trailed his leading rivals, admitting disappointment and vowing to catch up. Bu the McCain campaign brought in just $7 million in April and May, one person familiar with the campaign's internal tallies said, speaking anonymously because the totals are confidential. + Trying to manage expectations, his advisers are saying publicly that their only goal is to exceed the $12.5 million Mr. McCain brought in the first quarter. But that result would do nothing to narrow the gap behind the growing war chests of his main rivals for his party's nomination, former Gov. Mitt Romney of Massachusetts and former Mayor Rudolph W. Giuliani of New York, who expect to exceed their first quarter totals as well. + The sprint for big checks is a stark contrast to his insurgent campaign for the Republican nomination eight years ago, when Mr. McCain relied more than any candidate on anonymous small donors mailing in checks. Out of about $29 million in total contributions to his 2000 campaign, about a third of his donors gave less than $200 and fewer than about a third gave more than $1,000, according to the Center for Responsive Politics. + Those small donors applauded his attacks on special interests, but the same crusades can be liabilities with the big Republican fund-raisers, contributors and political action committees he is courting to remain competitive, his advisers say. + All campaigns play up their fund-raising challenges in order to cast their ultimate totals in the best light, but both campaign finance records and the views of Washington lobbyists suggest that Mr. McCain has consistently turned off the constituencies that have been expected to make up his financial base. + The military industry and its lobbyists, for example, always give heavily to the handful of lawmakers who control military spending; that is what fuels their business. + Last year, for example, the industry gave $244,000 to the chairman of the House armed services committee at the time, Representative Duncan Hunter, even though he faced scant opposition in his California district. + But military industry lobbyists say they have long dreaded the prospect that Mr. McCain might ascend to the chairmanship of the committee, much less the presidency. He is the Senate's most outspoken critic of military procurement policies, big Pentagon contracts, and especially earmarks -- the Congressional add-ons to military spending bills that contractors crave. + ''McCain has been a one-man wrecking crew,'' said a prominent military lobbyist, speaking on condition of anonymity because he lobbies the committee. + Mr. McCain has sometimes chalked up his fund-raising problems to his own lack of enthusiasm for the asking for money. ''When you don't like to do something,'' he told reporters on an April bus trip through New Hampshire shortly after the disclosure of his disappointing first quarter, letting his voice trail off. ''But we're doing better. I'm working a lot harder at it. We'll do better. We'll do a lot better.'' + If Mr. McCain feels any pressure, he does not show it, some fund-raisers said. Minutes before the start of a $2,300-a-head fund-raiser last Sunday at the Sonoma Valley home of the venture capitalist J. Gary Shansby, Mr. McCain was picking and eating berries as though he had not a care in the world, said Mr. Shansby, who has committed to raise $1 million. + Several fund-raisers said Mr. McCain was no more likely to tailor his messages to his donors in person than he has been in the Senate. + ''I tell him all the time: 'Everyone knows where you are on Iraq. Let's talk about the environment, pork barrel spending, health care, dependence on foreign oil,' '' Mr. Shansby said, reflecting the concerns of wealthy Sonoma Valley Republicans -- many of whom are skeptical of the war. + Mr. McCain nonetheless made his support for the war the centerpiece of his fund-raiser. ''He gets very agitated about a few issues,'' Mr. Shansby said, ''the war, immigration reform, campaign finance.'' + POLITICAL MEMO \ No newline at end of file diff --git a/run-groovy.bat b/run-groovy.bat new file mode 100644 index 0000000..752c74f --- /dev/null +++ b/run-groovy.bat @@ -0,0 +1,43 @@ +@echo off + +if "%1"=="" ( + echo "Usage: %0 ..." + exit +) + +set PROGRAM=%1 +if not exist "%PROGRAM%" ( + echo "Not found:" $PROGRAM + exit +) + +set GROOVY=C:\Users\%USERNAME%\Desktop\Java\groovy-1.8.9 +set GREMLIN_JARS=C:\Users\%USERNAME%\Desktop\Java\gremlin-groovy-2.4.0\lib\* +set VIZLINCDB_JARS=C:\Users\%USERNAME%\Documents\NetBeansProjects\vizlincdb\target\* +set VIZLINCDB_LIB_JARS=C:\Users\%USERNAME%\Documents\NetBeansProjects\vizlincdb\target\lib\* + +set LIB_JARS=lib\* +set GROOVY_ALL_JAR=%GROOVY%\embeddable\groovy-all-1.8.9.jar +set SRC=src + +REM GROOVY_ALL_JAR should be before GREMLIN_JARS. GREMLIN_JARS includes the non-"all" groovy jar +set CP=%SRC%;%GROOVY_ALL_JAR%;%GREMLIN_JARS%;%LIB_JARS%;%VIZLINCDB_JARS%;%VIZLINCDB_LIB_JARS% + +set GC_DEBUG=0 +if "%GC_DEBUG"=="1" ( + set JAVA_OPTS="-d64 -server -Xmx2g -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails" +) else ( + set JAVA_OPTS="-d64 -server -Xmx2g -XX:+UseConcMarkSweepGC" +) + +REM Get the rest of the args into a single variable. +shift +set REMAINING_ARGS= +:start + if "%1"=="" goto done + set REMAINING_ARGS=%REMAINING_ARGS% %1 + shift + goto start +:done + +java -cp %CP% groovy.ui.GroovyMain "%PROGRAM%" %REMAINING_ARGS% diff --git a/run-groovy.sh b/run-groovy.sh new file mode 100755 index 0000000..fa58756 --- /dev/null +++ b/run-groovy.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Set the exit status to the first non-zero status in a pipeline, so if any command in the pipe fails, we'll know. +set -o pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage:" $_ " ..." + exit 1 +fi +PROGRAM=$1 +PROGRAM_BASENAME=$(basename ${PROGRAM} .groovy) +shift + +TIMESTAMP=`date +'%Y%m%d-%H%M%S'` + +if [[ ! -e $PROGRAM ]]; then + echo "Not found:" $PROGRAM + exit 1 +fi + +OS=$(uname -o) +if [[ ${OS} == "Cygwin" ]] ; then PLATFORM=Cygwin; fi +GROOVY=${HOME}/code/groovy-1.8.9 +GREMLIN_JARS=${HOME}'/code/gremlin-groovy-2.4.0/lib/*' +VIZLINCDB_JARS='../vizlincdb/target/*' +VIZLINCDB_LIB_JARS='../vizlincdb/target/lib/*' + +if [[ ${OS} == "Cygwin" ]] ; +then + GROOVY=C:/Users/${USER}/Desktop/Java/groovy-1.8.9 + GREMLIN_JARS=C:/Users/${USER}'/Desktop/Java/gremlin-groovy-2.4.0/lib/*' + VIZLINCDB_JARS=C:/Users/${USER}'/Documents/NetBeansProjects/vizlincdb/target/*' + VIZLINCDB_LIB_JARS=C:/Users/${USER}'/Documents/NetBeansProjects/vizlincdb/target/lib/*' +fi + +LIB_JARS='lib/*' +GROOVY_ALL_JAR=${GROOVY}/embeddable/groovy-all-1.8.9.jar + +SRC='src' +# GROOVY_ALL_JAR should be before GREMLIN_JARS. GREMLIN_JARS includes the non-"all" groovy jar +CP=${SRC}:${GROOVY_ALL_JAR}:${GREMLIN_JARS}:${LIB_JARS}:${VIZLINCDB_JARS}:${VIZLINCDB_LIB_JARS}:${CLASSPATH} + +# Change path delimiters to ';' on Windows +if [[ ${OS} == "Cygwin" ]] ; +then + # Must quote due to semicolons. + CP="${SRC};${GROOVY_ALL_JAR};${GREMLIN_JARS};${LIB_JARS};${VIZLINCDB_JARS};${VIZLINCDB_LIB_JARS};${CLASSPATH}" +fi + +GC_DEBUG=0 +if [ "${GC_DEBUG}" = "1" ] ; then + export JAVA_OPTS="-d64 -server -Xmx12g -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails" +else + export JAVA_OPTS="-d64 -server -Xmx12g -XX:+UseConcMarkSweepGC" +fi + +mkdir -p logs +# Use pipefail (see above) so failing status will be caught. + +# The groovy command uses the groovy jar, not the groovy-all jar, so the other jars it needs (from groovy's lib directory) may conflict. +# Avoid this problem by running the GroovyMain and using the groovy-all jarjar that includes everything groovy depends on. +java -cp ${CP} groovy.ui.GroovyMain ${PROGRAM} "$@" | tee logs/${PROGRAM_BASENAME}.${TIMESTAMP}.txt diff --git a/src/AcrossDocLocCoref.groovy b/src/AcrossDocLocCoref.groovy new file mode 100644 index 0000000..da6dd00 --- /dev/null +++ b/src/AcrossDocLocCoref.groovy @@ -0,0 +1,117 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* +import com.wcohen.ss.* + + +// Across document coref for person tags + +class AcrossDocLocCoref extends CorefBase { + + AcrossDocLocCoref(VizLincDB graphdb) { + super(graphdb) + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: AcrossDocLocCoref.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + AcrossDocLocCoref coref = new AcrossDocLocCoref(db) + coref.findCoreferences() + db.shutdown() + } + + + def isCandidate (name, num_tokens_min, num_tokens_max, num_chars_min) { + if (name.length() < num_chars_min) + return false + + def num_tokens = name.split().size() + if ((num_tokens < num_tokens_min) || (num_tokens > num_tokens_max)) + return false + + return true + } + + + void findCoreferences() { + + // Tunable parameters + final NUM_TOKENS_MIN = 1 + final NUM_TOKENS_MAX = 10 + final NUM_CHARS_MIN = 2 + final MIN_NUM_DOCS = 2 + def DIST = new Levenstein() + final THRESH = -1.1 + + final DEBUG = false + + // Query for location nodes + def nodes = db.getEntitiesOfType(E_LOCATION) + + // First pass -- get potential merge candidates + def strong_candidates_list = [] + def weak_candidates_list = [] + def i = 0 + for (node in nodes) { + def text = node[P_ENTITY_TEXT] + def nodeAndText = new NodeAndText(node, text) + if (isCandidate(text, NUM_TOKENS_MIN, NUM_TOKENS_MAX, NUM_CHARS_MIN)) { + strong_candidates_list.add(nodeAndText); + } else { + weak_candidates_list.add(nodeAndText); + } + i += 1 + if ((i % 10000) == 0) println "At entity #: " + i + } + strong_candidates_list = strong_candidates_list.sort{it.text} + weak_candidates_list = weak_candidates_list.sort{it.text} + + // Create clusters + println "Length of strong_candidates_list: " + strong_candidates_list.size() + def strong_clusters = combineByExactMatch (strong_candidates_list) + println "Length of weak_candidates_list: " + weak_candidates_list.size() + def weak_clusters = combineByExactMatch(weak_candidates_list) + println "Exact match, number of strong clusters: " + strong_clusters.size() + println "Exact match, number of weak clusters: " + weak_clusters.size() + combineAdjacentClose(strong_clusters, DIST, THRESH, true) + println "Close match, number of strong clusters: " + strong_clusters.size() + + if (DEBUG) { + println "Strong clusters:" + output_clusters(strong_clusters) + println "\nWeak clusters:" + output_clusters(weak_clusters) + } + + + // Filter strong clusters. Weak clusters are not filtered based on MIN_NUM_DOCS. + def remove_list = [] + for (c in strong_clusters) { + def sz = c.value.size() + if (sz < MIN_NUM_DOCS) { + remove_list.add(c.key) + } + } + for (ky in remove_list) { + strong_clusters.remove(ky) + } + + println "Number of remaining strong clusters = " + strong_clusters.size() + + // Merge clusters + println "Merging clusters" + mergeClusters(strong_clusters, 'across_doc_location_coref') + mergeClusters(weak_clusters, 'weak_across_doc_location_coref') + + db.commit() + } +} + diff --git a/src/AcrossDocOrgCoref.groovy b/src/AcrossDocOrgCoref.groovy new file mode 100644 index 0000000..da799b5 --- /dev/null +++ b/src/AcrossDocOrgCoref.groovy @@ -0,0 +1,116 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* +import com.wcohen.ss.* + +// Across document coref for person tags + +class AcrossDocOrgCoref extends CorefBase { + + AcrossDocOrgCoref(VizLincDB graphdb) { + super(graphdb) + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: AcrossDocOrgCoref.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + AcrossDocOrgCoref coref = new AcrossDocOrgCoref(db) + coref.findCoreferences() + db.shutdown() + } + + def isCandidate (name, num_tokens_min, num_tokens_max, num_chars_min) + { + if (name.length() < num_chars_min) + return false + + def num_tokens = name.split().size() + if ((num_tokens < num_tokens_min) || (num_tokens > num_tokens_max)) + return false + + return true + } + + + void findCoreferences() { + // Tunable parameters + final NUM_TOKENS_MIN = 1 + final NUM_TOKENS_MAX = 20 + final NUM_CHARS_MIN = 2 + final MIN_NUM_DOCS = 2 + def DIST = new Levenstein() + final THRESH = -1.1 + + def DEBUG = false + + // Query for organization nodes + def nodes = db.getEntitiesOfType(E_ORGANIZATION) + + // First pass -- get potential merge candidates + def strong_candidates_list = [] + def weak_candidates_list = [] + def i = 0 + for (node in nodes) { + def text = node[P_ENTITY_TEXT] + def nodeAndText = new NodeAndText(node, text) + if (isCandidate(text, NUM_TOKENS_MIN, NUM_TOKENS_MAX, NUM_CHARS_MIN)) { + strong_candidates_list.add(nodeAndText); + } else { + weak_candidates_list.add(nodeAndText); + } + i += 1 + if ((i % 10000) == 0) println "At entity #: " + i + } + strong_candidates_list = strong_candidates_list.sort{it.text} + weak_candidates_list = weak_candidates_list.sort{it.text} + + // Create clusters + println "Length of strong_candidates_list: " + strong_candidates_list.size() + def strong_clusters = combineByExactMatch(strong_candidates_list) + println "Length of weak_candidates_list: " + weak_candidates_list.size() + def weak_clusters = combineByExactMatch(weak_candidates_list) + println "Exact match, number of strong clusters: " + strong_clusters.size() + println "Exact match, number of weak clusters: " + weak_clusters.size() + combineAdjacentClose(strong_clusters, DIST, THRESH, true) + println "Close match, number of strong clusters: " + strong_clusters.size() + + if (DEBUG) { + println "Strong clusters:" + output_clusters(strong_clusters) + println "\nWeak clusters:" + output_clusters(weak_clusters) + } + + + // Filter strong clusters. Weak clusters are not filtered based on MIN_NUM_DOCS. + def remove_list = [] + for (c in strong_clusters) { + def sz = c.value.size() + if (sz < MIN_NUM_DOCS) { + remove_list.add(c.key) + } + } + for (ky in remove_list) { + strong_clusters.remove(ky) + } + + println "Number of remaining strong clusters = " + strong_clusters.size() + + // Merge clusters + println "Merging clusters" + mergeClusters(strong_clusters, 'across_doc_organization_coref') + mergeClusters(weak_clusters, 'weak_across_doc_organization_coref') + + db.commit() + } + +} + diff --git a/src/AcrossDocPerCoref.groovy b/src/AcrossDocPerCoref.groovy new file mode 100644 index 0000000..a425602 --- /dev/null +++ b/src/AcrossDocPerCoref.groovy @@ -0,0 +1,119 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* +import com.wcohen.ss.* + +// Across document coref for person tags + +class AcrossDocPerCoref extends CorefBase { + + AcrossDocPerCoref(VizLincDB graphdb) { + super(graphdb) + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: AcrossDocPerCoref.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + AcrossDocPerCoref coref = new AcrossDocPerCoref(db) + coref.findCoreferences() + db.shutdown() + } + + + def isCandidate (name, num_tokens_min, num_tokens_max, num_chars_min) { + if (name.length() < num_chars_min) + return false + + def num_tokens = name.split().size() + if ((num_tokens < num_tokens_min) || (num_tokens > num_tokens_max)) + return false + + if (!(name==~/[A-Za-z\.\s]+/)) + return false + + return true + } + + + void findCoreferences() { + // Tunable parameters + final NUM_TOKENS_MIN = 2 + final NUM_TOKENS_MAX = 6 + final NUM_CHARS_MIN = 5 + final MIN_NUM_DOCS = 2 + def DIST = new Levenstein() + final THRESH = -1.1 + + final DEBUG = false + + // Query for entity nodes + def nodes = db.getEntitiesOfType(E_PERSON) + + // First pass -- get potential merge candidates + def strong_candidates_list = [] + def weak_candidates_list = [] + def i = 0 + for (node in nodes) { + def text = node[P_ENTITY_TEXT] + def nodeAndText = new NodeAndText(node, text) + if (isCandidate(text, NUM_TOKENS_MIN, NUM_TOKENS_MAX, NUM_CHARS_MIN)) { + strong_candidates_list.add(nodeAndText); + } else { + weak_candidates_list.add(nodeAndText); + } + i += 1 + if ((i % 10000) == 0) println "At entity #: " + i + } + + strong_candidates_list = strong_candidates_list.sort{it.text} + weak_candidates_list = weak_candidates_list.sort{it.text} + + // Create clusters + println "Length of strong_candidates_list: " + strong_candidates_list.size() + def strong_clusters = combineByExactMatch (strong_candidates_list) + println "Length of weak_candidates_list: " + weak_candidates_list.size() + def weak_clusters = combineByExactMatch(weak_candidates_list) + println "Exact match, number of strong clusters: " + strong_clusters.size() + println "Exact match, number of weak clusters: " + weak_clusters.size() + combineAdjacentClose(strong_clusters, DIST, THRESH, true) + println "Close match, number of strong clusters: " + strong_clusters.size() + + if (DEBUG) { + println "Strong clusters:" + outputClusters(strong_clusters) + println "\nWeak clusters:" + outputClusters(weak_clusters) + } + + + // Filter strong clusters. Weak clusters are not filtered based on MIN_NUM_DOCS. + def remove_list = [] + for (c in strong_clusters) { + def sz = c.value.size() + if (sz < MIN_NUM_DOCS) { + remove_list.add(c.key) + } + } + for (ky in remove_list) { + strong_clusters.remove(ky) + } + + println "Number of remaining strong clusters = " + strong_clusters.size() + + // Merge clusters + println "Merging clusters" + mergeClusters (strong_clusters, 'across_doc_person_coref') + mergeClusters (weak_clusters, 'weak_across_doc_person_coref') + + db.commit() + } + +} \ No newline at end of file diff --git a/src/AcrossDocSimpleCoref.groovy b/src/AcrossDocSimpleCoref.groovy new file mode 100644 index 0000000..41159b1 --- /dev/null +++ b/src/AcrossDocSimpleCoref.groovy @@ -0,0 +1,105 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +// Across document coref for given entity type + +class AcrossDocSimpleCoref extends CorefBase { + + String entityType // One of E_PERSON, E_LOCATION, etc. + + AcrossDocSimpleCoref(String entityType, VizLincDB graphdb) { + super(graphdb) + this.entityType = entityType + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 2) { + println "Usage: AcrossDocSimpleCoref.groovy " + System.exit(1) + } + def entityType = args[0] + def db = new VizLincDB(args[1]) + AcrossDocSimpleCoref coref = new AcrossDocSimpleCoref(entityType, db) + coref.findCoreferences() + db.shutdown() + } + + + def isCandidate (name, num_tokens_min, num_tokens_max, num_chars_min) { + if (name.length() < num_chars_min) + return false + + def num_tokens = name.split().size() + if ((num_tokens < num_tokens_min) || (num_tokens > num_tokens_max)) + return false + + if (!(name==~/[A-Za-z\.\s]+/)) + return false + + return true + } + + void findCoreferences(Boolean useGlobalID=false) { + final MIN_NUM_DOCS = 2 + final DEBUG = false + + println "In findCoreferences for AcrossDoc stuff for entity type " + entityType + " ..." + + // Query for entity nodes + def nodes = db.getEntitiesOfType(entityType) + + // First pass -- get potential merge candidates + def candidatesList = [] + def i = 0 + for (node in nodes) { + def text = node.getProperty(P_ENTITY_TEXT) + def nodeAndText = new NodeAndText(node, text) + candidatesList.add(nodeAndText); + i += 1 + if ((i % 10000) == 0) println "At entity #: " + i + } + candidatesList = candidatesList.sort{it.text} + + // Create clusters + println "Length of candidatesList: " + candidatesList.size() + def clusters + if (useGlobalID) { + clusters = combineByGlobalID(candidatesList) + } else { + clusters = combineByExactMatch(candidatesList) + } + println "Exact match, number of clusters: ${clusters.size()}" + + if (DEBUG) { + println "clusters:" + outputClusters(clusters) + } + + // Filter clusters based on MIN_NUM_DOCS. + def remove_list = [] + for (c in clusters) { + def sz = c.value.size() + if (sz < MIN_NUM_DOCS) { + remove_list.add(c.key) + } + } + for (ky in remove_list) { + clusters.remove(ky) + } + + println "Number of remaining clusters = " + clusters.size() + + // Merge clusters + println "Merging clusters" + // mergeClusters (clusters, "across_doc_${entityType.toLowerCase()}_coref".toString()) + mergeClusters (clusters, "across_doc_" + entityType.toLowerCase() + "_coref") + db.commit() + } + +} \ No newline at end of file diff --git a/src/CorefBase.groovy b/src/CorefBase.groovy new file mode 100644 index 0000000..1691284 --- /dev/null +++ b/src/CorefBase.groovy @@ -0,0 +1,279 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import java.text.Normalizer +import java.text.Normalizer.Form +import com.wcohen.ss.* + +abstract class CorefBase { + + // Fields + VizLincDB db + + CorefBase(graphdb) { + db = graphdb + } + + def addEntitiesToGraph (clusters, entity_type, created_by) { + for (j in clusters) { + def name = j.key + def mentions = j.value + // println "name: $name, mentions: $mentions" + def node = db.newEntity(entity_type, name, created_by) + node.setProperty('num_mentions',mentions.size()) + node.setProperty('num_docs', 1) + db.connectEntityToMentionsAndDocuments(node, mentions.collect{it.node}) + } + db.commit() + } + + def addEntitiesToGraphByGlobalID (clusters, entity_type, created_by) { + for (j in clusters) { + def gid = j.key + def mentions = j.value + def name = mentions[0].text // All the same + // println "name: $name, mentions: $mentions" + def node = db.newEntity(entity_type, name, created_by) + node.setProperty(P_ENTITY_GLOBAL_ID, gid) + node.setProperty('num_mentions',mentions.size()) + node.setProperty('num_docs', 1) + db.connectEntityToMentionsAndDocuments(node, mentions.collect{it.node}) + } + db.commit() + } + + final static JUNK_CHARS_TO_REMOVE = /[\^\"\<\>_]/ + // " is a good char in a loc + final static JUNK_CHARS_TO_REMOVE_FOR_LOCS = /[\^\<\>_]/ + + // Return a list of NodeAndText objects. Each NodeAndText object references a node and also extracts the current value of + // the P_MENTION_TEXT property. The texts are extracted so that they may be modified later. + // The mention texts are normalized and then the list is sorted by the text. Mention texts that become empty after + // normalization are dropped. + def getNormalizedMentions(doc_node, node_type, junkToRemove = JUNK_CHARS_TO_REMOVE) { + return db.getMentionsInDocumentOfType(doc_node, node_type).toList().collect { node -> + def text = normalizeStr(node.getProperty(P_MENTION_TEXT), junkToRemove) + return text ? new NodeAndText(node, text) : null + }.grep{it}.sort{it.text} + } + + // As above, but without normalization (assume they are already normalized) + def getTwitterNormalizedMentions(doc_node, node_type) { + return db.getMentionsInDocumentOfType(doc_node, node_type).toList().collect { node -> + def text = twitterNormalizeStr(node.getProperty(P_MENTION_TEXT)) + return text ? new NodeAndText(node, text) : null + }.grep{it}.sort{it.text} + } + + // As above, but without normalization (assume they are already normalized) + def getMentions(doc_node, node_type) { + return db.getMentionsInDocumentOfType(doc_node, node_type).toList().collect { node -> + def text = node.getProperty(P_MENTION_TEXT) + return text ? new NodeAndText(node, text) : null + }.grep{it}.sort{it.text} + } + + def normalizeStr (String str, junkToRemove) { + def NON_WORD = '[^A-Za-z0-9-]' + def out = str.trim().toUpperCase() + // Remove accents. + out = Normalizer.normalize(out, Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+","") + // Normalize whitespace to single spaces. + out = out.replaceAll(/[\s]+/, " ") + // Remove leading and trailing non-word chars. + out = out.replaceAll("^$NON_WORD+", '').replaceAll("$NON_WORD+\$", '') + // Replace junk chars + out = out.replaceAll(junkToRemove, " ") + return out + } + + def twitterNormalizeStr (String str) { + def out = str.trim().toLowerCase() + out = Normalizer.normalize(out, Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+","") + return out + } + + def combineFirstnameFullname (clusters) { + + def min_num_chars = 2 + + // First pass -- find candidate first names + // Look for followed by , where "first name" is loosely interpreted + def first_names = [:] + def first_names_full = [:] + def last = "" + def curr_first_name = null + for (j in clusters) { + def curr = j.key + def min_len = [curr.length(),last.length()].min() + if (last != "" && last == curr.substring(0,min_len) && min_len>=min_num_chars) { // first names at least 2 characters + curr_first_name = last + first_names[last] = 1 + first_names_full[last] = curr + } else if (curr_first_name != null){ // check to see how many subsequent names have this "first name" + min_len = [curr.length(),curr_first_name.length()].min() + if (curr.substring(0,min_len)==curr_first_name) { + first_names[curr_first_name] += 1 + } + } + last = j.key + } + + // Second pass -- if there's exactly one match, then combine. Otherwise, this first name may appear multiple times. + def num_combined = 0 + for (fn_map in first_names) { + def fn = fn_map.key + def fn_full = first_names_full[fn] + if (fn_map.value == 1) { + clusters[fn_full] = clusters[fn_full] + clusters[fn] + clusters.remove(fn) + num_combined += 1 + } + } + + } + + def combineByExactMatch (in_list) { + // Take a name list and produce the first set of clusters using exact string match + def out_map = [:] // Default hash map in groovy is linked, so order is maintained + def last = null + def curr_list = null + for (it in in_list) { + if (it.text==last) { + curr_list.add(it) + } else { + if (curr_list != null) { + out_map[last] = curr_list + } + curr_list = [it] + } + last = it.text + } + if (curr_list!=null) { + out_map[last] = curr_list + } + return out_map + } + + def combineByGlobalID (in_list) { + // Take a name list and produce the first set of clusters using only global IDs + def out_map = [:] // Default hash map in groovy is linked, so order is maintained + + for (it in in_list) { + def gid = it.node[P_MENTION_GLOBAL_ID] + if (gid) { + if (!out_map.containsKey(gid)) + out_map[gid] = [] + out_map[gid].add(it) + } + } + return out_map + } + + static final Levenstein LEVENSTEIN = new Levenstein() + // def static LEVEL2JAROWINKLER = new Level2JaroWinkler() + // def static LEVEL2JAROWINKLER_THRESH = 0.9 + + def combineAdjacentClose (clusters, dist=LEVENSTEIN, thresh=-1.1, match_numbers_exactly=true) { + // Combine two names if they are close both in edit distance and in a sorted list + // First character OCR errors are a problem + + def candidate_merge = [:] + def last = "" + + // Find merge candidates + for (j in clusters) { + def curr = j.key + def d = dist.score(last, curr) + if (last!="" && dist.score(last, curr) > thresh && (!match_numbers_exactly || allNumbersMatch(last, curr))) { + candidate_merge[curr] = last + // Uncomment to check merge criteria. + // println "yesmerge: $d\t$last ::: $curr" // XXX + } else { + // Uncomment to check merge criteria. + // println "nomerge: $d\t$last ::: $curr" // XXX + } + last = curr + } + + // Now merge + for (candidate in candidate_merge) { + def nm1 = candidate.key + def nm2 = candidate.value + if (clusters.containsKey(nm1) && clusters.containsKey(nm2)) { + if (clusters[nm2].size() > clusters[nm1].size()) { // Pick the cluster with the most elements + clusters[nm2] = clusters[nm2] + clusters[nm1] + clusters.remove(nm1) + } else { + clusters[nm1] = clusters[nm1] + clusters[nm2] + clusters.remove(nm2) + } + } + } + } + + def static NUMBER_PATTERN = ~ /\d+/ + // Make sure any sequences of digits in s and t match exactly. + def allNumbersMatch(s, t) { + return ((s =~ NUMBER_PATTERN).collect{it}) == ((t =~ NUMBER_PATTERN).collect{it}) + } + + def mergeClusters (clusters, created_by) { + def i = 0 + println "Edges before mergeClusters: ${db.graph.E.count()}" + for (c in clusters) { + if (((i%100) == 0) || (c.value.size() > 100)) { + println "Merging cluster: " + i + " with size " + c.value.size() + } + + // Find most common name -- largest subcluster + def names = [:] + c.value.each{ names[it.text]=0 } + c.value.each{ names[it.text] += 1 } + def most_common_name = names.max{ it.value }.key + def most_common_node = c.value.find{ it.text==most_common_name }.node + most_common_node.setProperty(P_CREATED_BY, created_by) + + // Loop over nodes in cluster and merge + + def num = 0 + for (pr in c.value) { + def node = pr.node + if (node!=most_common_node) { + // Redo edges to most common node and delete + most_common_node.setProperty('num_mentions', + most_common_node.getProperty('num_mentions') + + node.getProperty('num_mentions')) + most_common_node.setProperty('num_docs', most_common_node.getProperty('num_docs') + 1) + // .toList() to get the whole list because otherwise the + // connectEntityToMentionsAndDocuments() will add new edges + // that will be picked up by the Gremlin pipeline while it is still + // iterating. + def mention_nodes = node.inE(L_MENTION_TO_ENTITY).outV.toList() + db.connectEntityToMentionsAndDocuments(most_common_node, mention_nodes) + db.deleteNode(node) + num += 1 + if (num > 1000) { + db.commit() + num = 0 + } + } + } + + if (((i%100) == 0) || (c.value.size() > 50)) { + db.commit() + } + + i += 1 + } + db.commit() + println "Edges after mergeClusters: ${db.graph.E.count()}" + } + + def outputClusters (clusters) { + println "Clusters are (" + clusters.size() + "):" + for (j in clusters) { + println j.key + " " + j.value.size() + " " + j.value + } + } + +} \ No newline at end of file diff --git a/src/Countries.groovy b/src/Countries.groovy new file mode 100644 index 0000000..3ff31ba --- /dev/null +++ b/src/Countries.groovy @@ -0,0 +1,304 @@ +class Countries { + // Returns null if country not found. + public static List lookup(String country) { + return COUNTRIES_LAT_LONG.get(country) + } + + // Return true if the given string ends with any of the countries. + // Assume input string is upper-cased and trimmed. + public static boolean endsWithACountry(String location) { + for (String country : COUNTRIES_LAT_LONG.keySet()) { + // Add a space to avoid non-token mismatches, e.g. "LAO" matching "CALLAO". + if (location.endsWith(' ' + country)) { + return true + } + } + return false + } + + static final COUNTRIES_LAT_LONG = [ + "AFGANISTAN" : [33f, 65f], + "ISLAS ALAND" : [60.25f, 20f], + "ALBANIA" : [41f, 20f], + "ARGELIA" : [28f, 3f], + "SAMOA" : [-14.3333333333f, -170f], + "SAMOA AMERICANA" : [-14.3333333333f, -170f], + "ANDORRA" : [42.5f, 1.5f], + "ANGOLA" : [-12.5f, 18.5f], + "ANGUILA" : [18.25f, -63.1666666667f], + "ANTARTIDA" : [-90f, 0f], + "ANTIGUA Y BARBUDA" : [17.05f, -61.8f], + "ANTIGUA" : [17.05f, -61.8f], + "BARBUDA" : [17.05f, -61.8f], + "ARGENTINA" : [-34f, -64f], + "ARMENIA" : [40f, 45f], + "ARUBA" : [12.5f, -69.9666666667f], + "AUSTRALIA" : [-27f, 133f], + "AUSTRIA" : [47.3333333333f, 13.3333333333f], + "AZERBAIYAN" : [40.5f, 47.5f], + "BAHAMAS" : [24.25f, -76f], + "BAHREIN" : [26f, 50.55f], + "BANGLADESH" : [24f, 90f], + "BARBADOS" : [13.1666666667f, -59.5333333333f], + "BELARUS" : [53f, 28f], + "BELGICA" : [50.8333333333f, 4f], + "BELICE" : [17.25f, -88.75f], + "BENIN" : [9.5f, 2.25f], + "BERMUDA" : [32.3333333333f, -64.75f], + "BHUTAN" : [27.5f, 90.5f], + "BOLIVIA" : [-17f, -65f], + "BONAIRE, SAN ESTAQUIO Y SABA" : [12.1666666667f, -68.25f], + "BONAIRE" : [12.1666666667f, -68.25f], + "BOSNIA Y HERZEGOVINA" : [44f, 18f], + "BOTSWANA" : [-22f, 24f], + "ISLA BOUVET" : [-54.4333333333f, 3.4f], + "BOUVET" : [-54.4333333333f, 3.4f], + "BRASIL" : [-10f, -55f], + "TERRITORIO BRITANICO DEL OCEANO INDICO" : [-6f, 71.5f], + "BRUNEI DARUSSALAM" : [4.5f, 114.666666667f], + "BRUNEI" : [4.5f, 114.666666667f], + "BULGARIA" : [43f, 25f], + "BURKINA FASO" : [13f, -2f], + "BURUNDI" : [-3.5f, 30f], + "CAMBOYA" : [13f, 105f], + "CAMERUN" : [6f, 12f], + "CANADA" : [60f, -95f], + "CABO VERDE" : [16f, -24f], + "ISLAS CAIMAN" : [19.5f, -80.5f], + "REPUBLICA CENTROAFRICANA" : [7f, 21f], + "CHAD" : [15f, 19f], + "CHILE" : [-30f, -71f], + "CHINA" : [35f, 105f], + "ISLA DE NAVIDAD" : [-10.5f, 105.666666667f], + "ISLAS COCOS [KEELING]" : [-12.5f, 96.8333333333f], + "COLOMBIA" : [4f, -72f], + "COMORAS" : [-12.1666666667f, 44.25f], + "CONGO" : [-1f, 15f], + "REPUBLICA DEMOCRATICA DEL CONGO" : [0.0166666667f, 25f], + "ISLAS COOK" : [-21.2333333333f, -159.766666667f], + "COSTA RICA" : [10f, -84f], + "COTE D'IVOIRE" : [8f, -5f], + "CROACIA" : [45.1666666667f, 15.5f], + "CUBA" : [21.5f, -80f], + "CURACAO" : [12.1666666667f, -69f], + "CHIPRE" : [35f, 33f], + "REPUBLICA CHECA" : [49.75f, 15.5f], + "DINAMARCA" : [56f, 10f], + "DJIBOUTI" : [11.5f, 43f], + "DOMINICA" : [15.4166666667f, -61.3333333333f], + "REPUBLICA DOMINICANA" : [19f, -70.6666666667f], + "ECUADOR" : [-2f, -77.5f], + "EGIPTO" : [27f, 30f], + "EL SALVADOR" : [13.8333333333f, -88.9166666667f], + "GUINEA ECUATORIAL" : [2f, 10f], + "ERITREA" : [15f, 39f], + "ESTONIA" : [59f, 26f], + "ETIOPIA" : [8f, 38f], + "ISLAS FALKLAND [MALVINAS]" : [-51.75f, -59f], + "ISLAS FALKLAND" : [-51.75f, -59f], + "MALVINAS" : [-51.75f, -59f], + "ISLAS FEROE" : [62f, -7f], + "FIJI" : [-18f, 175f], + "FINLANDIA" : [64f, 26f], + "FRANCIA" : [46f, 2f], + "GUAYANA FRANCESA" : [4f, -53f], + "POLINESIA FRANCESA" : [-15f, -140f], + "TERRITORIOS AUSTRALES FRANCESES" : [-37.8333333333f, 77.5333333333f], + "GABON" : [-1f, 11.75f], + "GAMBIA" : [13.4666666667f, -16.5666666667f], + "GEORGIA" : [42f, 43.5f], + "ALEMANIA" : [51f, 9f], + "GHANA" : [8f, -2f], + "GIBRALTAR" : [36.1333333333f, -5.35f], + "GRECIA" : [39f, 22f], + "GROENLANDIA" : [72f, -40f], + "GRANADA" : [12.1166666667f, -61.6666666667f], + "GUADALUPE" : [16.25f, -61.5833333333f], + "GUAM" : [13.4666666667f, 144.783333333f], + "GUATEMALA" : [15.5f, -90.25f], + "GUERNSEY" : [49.4666666667f, -2.5833333333f], + "GUINEA" : [11f, -10f], + "GUINEA-BISSAU" : [12f, -15f], + "GUYANA" : [5f, -59f], + "HAITI" : [19f, -72.4166666667f], + "ISLAS DE HEARD Y MCDONALD" : [-53.1f, 72.5166666667f], + "SANTA SEDE" : [41.9f, 12.45f], + "HONDURAS" : [15f, -86.5f], + "HONG KONG" : [22.25f, 114.166666667f], + "HUNGRIA" : [47f, 20f], + "ISLANDIA" : [65f, -18f], + "INDIA" : [20f, 77f], + "INDONESIA" : [-5f, 120f], + "IRAN" : [32, 53f], + "IRAQ" : [33f, 44f], + "IRLANDA" : [53f, -8f], + "ISLA DE MAN" : [54.25f, -4.5f], + "ISRAEL" : [31.5f, 34.75f], + "ITALIA" : [42.8333333333f, 12.8333333333f], + "JAMAICA" : [18.25f, -77.5f], + "JAPON" : [36f, 138f], + "JERSEY" : [49.25f, -2.1666666667f], + "JORDANIA" : [31f, 36f], + "KAZAJSTAN" : [48f, 68f], + "KENYA" : [1f, 38f], + "KIRIBATI" : [1.4166666667f, 173f], + "REPUBLICA POPULAR DEMOCRATICA DE COREA" : [40f, 127f], + "REPUBLICA DE COREA" : [37f, 127.5f], + "COREA" : [37f, 127.5f], + "KUWAIT" : [29.366667f, 47.966667f], + "KIRGUISTAN" : [41f, 75f], + "REPUBLICA DEMOCRATICA POPULAR LAO" : [18f, 105f], + "LAO" : [18f, 105f], + "LETONIA" : [57f, 25f], + "LIBANO" : [33.8333333333f, 35.8333333333f], + "LESOTHO" : [-29.5f, 28.5f], + "LIBERIA" : [6.5f, -9.5f], + "JAMAHIRIYA ARABE LIBIA" : [25f, 17f], + "LIECHTENSTEIN" : [47.2666666667f, 9.5333333333f], + "LITUANIA" : [56f, 24f], + "LUXEMBURGO" : [49.75f, 6.1666666667f], + "MACAO" : [22.1666666667f, 113.55f], + "EX REPUBLICA YUGOSLAVA DE MACEDONIA" : [41.8333333333f, 22f], + "MACEDONIA" : [41.8333333333f, 22f], + "MADAGASCAR" : [-20f, 47f], + "MALAWI" : [-13.5f, 34f], + "MALASIA" : [2.5f, 112.5f], + "MALDIVAS" : [3.25f, 73f], + "MALI" : [17f, -4f], + "MALTA" : [35.8333333333f, 14.5833333333f], + "ISLAS MARSHALL" : [9f, 168f], + "MARTINICA" : [14.6f, -61.0833333333f], + "MAURITANIA" : [20f, -12f], + "MAURICIO" : [-20.2833333333f, 57.55f], + "MAYOTTE" : [-12.8333333333f, 45.1666666667f], + "MEXICO" : [23f, -102f], + "MICRONESIA ESTADOS FEDERADOS DE" : [6.9166666667f,158.25f], + "MICRONESIA" : [6.9166666667f,158.25f], + "REPUBLICA DE MOLDOVA" : [47f, 29f], + "MOLDOVA" : [47f, 29f], + "MONACO" : [43.732778f, 7.419722f], + "MONGOLIA" : [46f, 105f], + "MONTENEGRO" : [42.5f, 19.3f], + "MONTSERRAT" : [16.75f, -62.2f], + "MARRUECOS" : [32f, -5f], + "MOZAMBIQUE" : [-18.25f, 35f], + "MYANMAR" : [22f, 98f], + "NAMIBIA" : [-22f, 17f], + "NAURU" : [-0.5333333333f, 166.916666667f], + "NEPAL" : [28f, 84f], + "PAISES BAJOS" : [52.5f, 5.75f], + "NUEVA CALEDONIA" : [-21.5f, 165.5f], + "NUEVA ZELANDIA" : [-41f, 174f], + "NICARAGUA" : [13f, -85f], + "NIGER" : [16f, 8f], + "NIGERIA" : [10f, 8f], + "NIUE" : [-19.0333333333f, -169.866666667f], + "ISLAS NORFOLK" : [-29.0333333333f, 167.95f], + "ISLAS MARIANAS SEPTENTRIONALES" : [15.2f, 145.75f], + "NORUEGA" : [62f, 10f], + "TERRITORI PALESTINA OCUPAT" : [32f, 35.25f], + "OMAN" : [21f, 57f], + "PAKISTAN" : [30f, 70f], + "PALAU" : [7.5f, 134.5f], + "PANAMA" : [9f, -80f], + "PAPUA NUEVA GUINEA" : [-6f, 147f], + "PARAGUAY" : [-23f, -58f], + "PERU" : [-10f, -76f], + "FILIPINAS" : [13f, 122f], + "PITCAIRN" : [-25.0666666667f, -130.1f], + "POLONIA" : [52f, 20f], + "PORTUGAL" : [39.5f, -8f], + "PUERTO RICO" : [18.25f, -66.5f], + "QATAR" : [25.5f, 51.25f], + "REUNION" : [-21.1f, 55.6f], + "RUMANIA" : [46f, 25f], + "FEDERACION DE RUSIA" : [60f, 100f], + "RWANDA" : [-2f, 30f], + "SAINT-BARTHELEMY" : [17.9f, -62.833333f], + "SANTA ELENA ASCENSION Y TRISTAN DA CUNHA" : [-15.95f,-5.7f], + "SANTA ELENA" : [-15.95f,-5.7f], + "ASCENSION" : [-15.95f,-5.7f], + "TRISTAN DA CUNHA" : [-15.95f,-5.7f], + "SAINT KITTS Y NEVIS" : [17.3333333333f, -62.75f], + "SAINT KITTS" : [17.3333333333f, -62.75f], + "NEVIS" : [17.3333333333f, -62.75f], + "SANTA LUCIA" : [13.8833333333f, -60.9666666667f], + "SAINT MARTIN [PARTE FRANCESA]" : [18.075278f, -63.06f], + "SAINT MARTIN" : [18.075278f, -63.06f], + "SAN PEDRO Y MIQUELON" : [46.8333333333f, -56.3333333333f], + "SAN VICENTE Y LAS GRANADINAS" : [13.25f, -61.2f], + "SAMOA" : [-13.5833333333f, -172.333333333f], + "SAN MARINO" : [43.933333f, 12.4166666667f], + "SANTO TOME Y PRINCIPE" : [1f, 7f], + "ARABIA SAUDITA" : [25f, 45f], + "SENEGAL" : [14f, -14f], + "SERBIA" : [44f, 21f], + "SEYCHELLES" : [-4.5833333333f, 55.6666666667f], + "SIERRA LEONA" : [8.5f, -11.5f], + "SINGAPUR" : [1.3666666667f, 103.8f], + "SAN MARTIN [PARTE HOLANDESA]" : [18.04167f, -63.06667f], + "SAN MARTIN" : [18.04167f, -63.06667f], + "REPUBLICA ESLOVACA" : [48.6666666667f, 19.5f], + "ESLOVACA" : [48.6666666667f, 19.5f], + "ESLOVENIA" : [46.1166666667f, 14.8166666667f], + "ISLAS SALOMON" : [-8f, 159f], + "SOMALI" : [10f, 49f], + "SUDAFRICA" : [-29f, 24f], + "GEORGIA DEL SUR E ISLAS SANDWICH DEL SUR" : [-54.5f, -37f], + "SUDAN DEL SUR" : [8f, 30f], + "ESPANA" : [40f, -4f], + "SRI LANKA" : [7f, 81f], + "SRILANKA" : [7f, 81f], + "SUDAN" : [15f, 30f], + "SURINAME" : [4f, -56f], + "ISLAS SVALBARD Y JAN MAYEN" : [78f, 20f], + "SVALBARD" : [78f, 20f], + "SWAZILANDIA" : [-26.5f, 31.5f], + "SUECIA" : [62f, 15f], + "SUIZA" : [47f, 8f], + "REPUBLICA ARABE SIRIA" : [35f, 38f], + "SIRIA" : [35f, 38f], + "PROVINCIA CHINA DE TAIWAN" : [23.5f, 121f], + "TAIWAN" : [23.5f, 121f], + "TAYIKISTAN" : [39f, 71f], + "REPUBLICA UNIDA DE TANZANIA" : [-6f, 35f], + "TANZANIA" : [-6f, 35f], + "TAILANDIA" : [15f, 100f], + "TIMOR-LESTE" : [-8.8333333333f, 125.916666667f], + "TOGO" : [8f, 1.1666666667f], + "TOKELAU" : [-9f, -172f], + "TONGA" : [-20f, -175f], + "TRINIDAD Y TABAGO" : [11f, -61f], + "TRINIDAD" : [11f, -61f], + "TABAGO" : [11f, -61f], + "TUNEZ" : [34f, 9f], + "TURQUIA" : [39f, 35f], + "TURKMENISTAN" : [40f, 60f], + "ISLAS TURCAS Y CAICOS" : [21.75f, -71.5833333333f], + "TURCAS Y CAICOS" : [21.75f, -71.5833333333f], + "TURCAS" : [21.75f, -71.5833333333f], + "TUVALU" : [-8f, 178f], + "UGANDA" : [1f, 32f], + "UCRANIA" : [49f, 32f], + "EMIRATOS ARABES UNIDOS" : [24f, 54f], + "EL REINO UNIDO DE GRAN BRETANA E IRLANDA DEL NORTE" : [54f, -2f], + "ESTADOS UNIDOS DE AMERICA" : [38f, -97f], + "ESTADOS UNIDOS" : [38f, -97f], + "ESTADOS UNIDOS ISLAS MENORES" : [16.75f, -169.516666667f], + "URUGUAY" : [-33f, -56f], + "UZBEKISTAN" : [41f, 64f], + "VANUATU" : [-16f, 167f], + "VENEZUELA" : [8f, -66f], + "VIET NAM" : [16.1666666667f, 107.833333333f], + "VIETNAM" : [16.1666666667f, 107.833333333f], + "ISLAS VIRGENES BRITANICAS" : [18.5f, -64.5f], + "ISLAS VIRGENES DE LOS ESTADOS UNIDOS" : [18.3333333333f, -64.8333333333f], + "ISLAS WALLIS Y FUTUNA" : [-13.3f, -176.2f], + "SAHARA OCCIDENTAL" : [24.5f, -13f], + "YEMEN" : [15f, 48f], + "ZAMBIA" : [-15f, 30f], + "ZIMBABWE" : [-20f, 30f] + ] + + static final COUNTRIES = COUNTRIES_LAT_LONG.keySet() + } \ No newline at end of file diff --git a/src/DateNormalizer.groovy b/src/DateNormalizer.groovy new file mode 100644 index 0000000..56e43c0 --- /dev/null +++ b/src/DateNormalizer.groovy @@ -0,0 +1,73 @@ +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses string and returns a NormalizedDate object corresponding to the input + * string. + * + * @author JO21372 + * adapted for Groovy: dhalbert + */ +class DateNormalizer { + static final LONG_MONTH = '(?:enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)' + static final SHORT_MONTH = '(?:ene|feb|mar|abr|may|jun|jul|ago|sep|oct|nov|dic)' + static final DAY_OF_MONTH = '(?:[1-9]|0[1-9]|[1-2][0-9]|30|31)' + static final DAY_OF_MONTH_WORDS = '(?:uno|primero|segundo|dos|tres|cuatro|cinco|seis|siete|ocho|nuevo|diez|once|doce|trece|catorce|quince|dieciseis|diecisiete|dieciocho|diecinueve|veinte|veintiuno|veintidos|veintitres|veinticuatro|veinticinco|veintiseis|veintisiete|veintiocho|veintinueve|treinta|treinta y uno)' + //Years in YYYY format from 1900 to 2012 + static final YEAR4 = /(?:1\.?9[0-9][0-9]|2\.?0[01][0-9])/ + static final YEAR2 = '(?:[0-9][0-9])' + + static normalizeString(String date) { + String result = date.toLowerCase(); + result = result.replaceAll(/\s+/, " "); + return result; + } + + static NormalizedDate normalize(String date) { + String nString = normalizeString(date); + NormalizedDate nDate = null; + + String regex1 = /($DAY_OF_MONTH) (?:de )?($LONG_MONTH) (?:de |del |del año )?($YEAR4)/ + String day = null; + String month = null; + String year = null; + + Pattern p1 = Pattern.compile(regex1); + Matcher m1 = p1.matcher(nString); + if (m1.find()) { + day = m1.group(1); + month = m1.group(2); + year = m1.group(3); + + nDate = new NormalizedDate(day, month, year); + return nDate; + } + + String regex2 = /(?:el día )?($DAY_OF_MONTH)([\/-])($SHORT_MONTH|$LONG_MONTH)\2($YEAR4|$YEAR2)/ + Pattern p2 = Pattern.compile(regex2); + Matcher m2 = p2.matcher(nString); + if (m2.find()) { + day = m2.group(1); + // group(2) is the / or - + month = m2.group(3); + year = m2.group(4); + nDate = new NormalizedDate(day, month, year); + return nDate; + } + + //Month first + String regex3 = /($LONG_MONTH) ($DAY_OF_MONTH) del? ($YEAR4)/ + Pattern p3 = Pattern.compile(regex3); + Matcher m3 = p3.matcher(nString); + if(m3.find()) { + day = m3.group(2); + month = m3.group(1); + year = m3.group(3); + nDate = new NormalizedDate(day, month, year); + return nDate; + } + + println(date + " -> " + nDate); + return nDate; + } +} \ No newline at end of file diff --git a/src/EdgeMentionsCounter.groovy b/src/EdgeMentionsCounter.groovy new file mode 100644 index 0000000..7de7ad8 --- /dev/null +++ b/src/EdgeMentionsCounter.groovy @@ -0,0 +1,84 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +// Add num_mentions property on document_entity edges + +class EdgeMentionsCounter { + + // Fields + VizLincDB db + + EdgeMentionsCounter(VizLincDB graphdb) { + db = graphdb + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: EdgeMentionsCounter.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + EdgeMentionsCounter emc = new EdgeMentionsCounter(db) + emc.countMentions() + db.shutdown() + } + + + void countMentions() { + def i = 0 + // def docs = db.getDocuments()._().order{it.a.getProperty(P_DOCUMENT_NAME) <=> it.b.getProperty(P_DOCUMENT_NAME)} + def docs = db.getDocuments() + + // Add in counts + for (doc in docs) { + i += 1 + if (i % 100 == 0) { + println "Committing at $i documents, at ${doc[P_DOCUMENT_NAME]}" + db.commit() + } + + // Get counts as assoc array + // def doc_entity_count = [:] + // doc.out(L_DOCUMENT_TO_ENTITY).transform{doc_entity_count[it.id]=0}.count() + // doc.out('document_to_mention').out('mention_to_entity').transform{doc_entity_count[it.id]++}.count() + + // Now save counts to graph + // doc.outE(L_DOCUMENT_TO_ENTITY).transform{it.inV.toList()[0][P_NUM_MENTIONS]=doc_entity_count[it.inV.toList()[0].id]}.count() + + // Get counts as assoc array and save to doc->entity edge (simpler version) + def doc_entity_count = doc.out(L_DOCUMENT_TO_MENTION).out(L_MENTION_TO_ENTITY).groupCount{it.id}.cap.next() + doc.outE(L_DOCUMENT_TO_ENTITY).sideEffect{def n=it.inV.toList()[0]; it[P_NUM_MENTIONS]=doc_entity_count[n.id]}.count() + } + println "Last commit..." + db.commit() + } + + void OLDcountMentions() { + def i = 0 + def docs = db.getDocuments()._().order{it.a.getProperty(P_DOCUMENT_NAME) <=> it.b.getProperty(P_DOCUMENT_NAME)} + for (doc in docs) { + i += 1 + if (i % 100 == 0) { + println "Committing at $i documents, at ${doc[P_DOCUMENT_NAME]}" + db.commit() + } + + def doc_entity_edges = doc.outE(L_DOCUMENT_TO_ENTITY) + for (doc_entity_edge in doc_entity_edges) { + def entity = doc_entity_edge.inV.toList()[0] // inV is the vertex that edge is going into (confusing terminology) + def matching_mentions_count = entity.in(L_MENTION_TO_ENTITY).in(L_DOCUMENT_TO_MENTION).filter{it == doc}.count() + // println "${doc[P_DOCUMENT_NAME]} ${entity[P_ENTITY_TEXT]} $matching_mentions_count" + doc_entity_edge.setProperty(P_NUM_MENTIONS, matching_mentions_count) + } + } + + println "Last commit..." + db.commit() + } +} \ No newline at end of file diff --git a/src/EntityExtractor.groovy b/src/EntityExtractor.groovy new file mode 100644 index 0000000..e1dcb53 --- /dev/null +++ b/src/EntityExtractor.groovy @@ -0,0 +1,69 @@ +import edu.stanford.nlp.ie.crf.CRFClassifier +import edu.stanford.nlp.util.Triple + +import edu.mit.ll.vizlincdb.io.CSVMention +import edu.mit.ll.vizlincdb.io.CSVMentionFileWriter + +class EntityExtractor { + + // fields + CRFClassifier classifier + + public EntityExtractor(File nerModelFile) { + classifier = CRFClassifier.getClassifier(nerModelFile) + } + + + static final DEFAULT_MODEL_NAME = 'ner-model.ser.gz' + + def extractEntitiesToCSV(File inputFile, File outputFile) { + String text = inputFile.getText('UTF-8') + CSVMentionFileWriter writer = new CSVMentionFileWriter(outputFile) + int entityIndex = 0 + for (Triple entityAndOffsets : classifier.classifyToCharacterOffsets(text)) { + // Stanford NER replaces regular spaces inside XML tags with Unicode non-breaking spaces. Undo this. + String mentionType = entityAndOffsets.first.replaceAll('\u00A0', ' ') + int start = entityAndOffsets.second + int stop = entityAndOffsets.third + String mentionText = text.substring(start, stop) + // We don't have a global id for the entity, so omit it (will read as NULL). + writer.write(new CSVMention(mentionType, start, stop, entityIndex, null, mentionText)) + entityIndex++ + } + writer.close(); + } + + + static main(String[] args) { + def cli = new CliBuilder(usage: 'EntityExtractor.groovy [options] file...') + cli.with { + h(longOpt: 'help', 'Show usage') + m(longOpt: 'model', required: false, args: 1, argName: 'model', "Stanford NER model filename (default: '${DEFAULT_MODEL_NAME}')") + o(longOpt: 'output-dir', args: 1, argName: 'outputDir', required: true, 'write extracted text files into this directory') + } + + def options = cli.parse(args) + if (!options) return + if (options.h) { + cli.usage() + return + } + def outputDir = new File(options.'output-dir') + def arguments = options.arguments() + if (arguments.size() == 0) { + cli.usage() + return + } + + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + + def entityExtractor = new EntityExtractor(new File(options.'model' ? options.'model' : DEFAULT_MODEL_NAME)); + for (filename in arguments) { + def inputFile = new File(filename) + println "Entity extraction: $filename" + entityExtractor.extractEntitiesToCSV(new File(filename), new File(outputDir, inputFile.getName() + '.ner.csv')) + } + } +} diff --git a/src/GeoLocation.groovy b/src/GeoLocation.groovy new file mode 100644 index 0000000..10235bf --- /dev/null +++ b/src/GeoLocation.groovy @@ -0,0 +1,16 @@ +class GeoLocation { + // Point values: + public Double lat + public Double lon + + // Bounding box values are returned from osm nominatim in the order below. + + public Double latS + public Double latN + public Double lonW + public Double lonE + + public String fullName // osm: displayName + public String osmType + +} diff --git a/src/Geocoder.groovy b/src/Geocoder.groovy new file mode 100644 index 0000000..ce07296 --- /dev/null +++ b/src/Geocoder.groovy @@ -0,0 +1,213 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import edu.mit.ll.vizlincdb.geo.GeoBoundingBox +import edu.mit.ll.vizlincdb.geo.GeoLocation + +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +import java.util.Collections +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class Geocoder { + + // Annotate location entities with latitude, longitude, possibly other info. + + final static NUM2 = /\d\d/ + final static NUM2_WITH_DECIMAL = /$NUM2(?:[.,]\d*)?/ + final static SEP = /\D{1,9}/ + final static NOISE = /\D*/ + final static LAT_LONG_1 = ~ /^$NOISE($NUM2)$SEP($NUM2)$SEP($NUM2_WITH_DECIMAL)$SEP($NUM2)$SEP($NUM2)$SEP($NUM2_WITH_DECIMAL)$NOISE$/ + final static LAT_LONG_2 = ~ /^$NOISE($NUM2)($NUM2)($NUM2)$SEP($NUM2)($NUM2)($NUM2)$NOISE$/ + + + static final BAD_COORDINATE = -999.0 + + // Fields + VizLincDB db + int locationCount = 0 + int numThreads + List geoResultList = Collections.synchronizedList(new ArrayList()) + OSMFetcher osmFetcher = null + + class GeoResult { + // Fields + def node + List locations + + GeoResult(node, locations) { + this.node = node + this.locations = locations + } + } + + Geocoder(VizLincDB db, numThreads=1) { + this.db = db + osmFetcher = new OSMFetcher() + + } + + + + String toDottedDecimal(num) { + return num.replaceAll(",", "."); + } + + // For running standalone. + static main(args) { + System.properties << [ 'http.proxyHost':'llproxy.llan.ll.mit.edu','http.proxyPort':'8080' ] + + Gremlin.load() + def cli = new CliBuilder(usage: 'Geocoder.groovy [options] db_name\n') + cli.with { + h longOpt: 'help', 'Show usage' + _ longOpt: 'coref-only', 'Use only locations that are marked with across_doc_location_coref' + } + + def options = cli.parse(args) + if (!options || options.h) { + cli.usage() + return + } + + + + // Must specify exactly one non-option. + def arguments = options.arguments() + if (arguments == null || arguments.size() > 1) { + println "ERROR: Specify exactly one database" + cli.usage() + return + } + + def dbname = arguments[0] + + def db = new VizLincDB(dbname) + Geocoder geocoder = new Geocoder(db) + geocoder.geocode(options.'coref-only') + db.shutdown() + } + + // Try to parse as a literal lat, long string + // Returns null if it can't parse -- need to lookup place name in this case + def parse_lat_long(place) { + // NUM could use . or , as a decimal point. + // Don't make SEP too long or we'll pick up bogus 6-number strings as lat-longs. + // All non-decimal coordinates are two digits. + def match = place =~ LAT_LONG_1 + if (!match.matches()) match = place =~ LAT_LONG_2 + // if (!match.matches()) match = place =~ LAT_LONG_3 // etc. + if (match.matches()) { + def groups = match[0] + def lat = groups[1].toDouble() + groups[2].toDouble()/60.0 + toDottedDecimal(groups[3]).toDouble()/3600.0 + def lon = -(groups[4].toDouble() + groups[5].toDouble()/60.0 + toDottedDecimal(groups[6]).toDouble()/3600.0) + return [new GeoLocation(lat, lon, null, place, null, null, null, GEO_SOURCE_COORDINATES)] + + } else { + return null + } + } + + def queueGeoLocations(node, geolocations) { + geoResultList.add(new GeoResult(node, geolocations)) + } + + def storeGeoLocations(geoResult) { + def node = geoResult.node + print node.getProperty('entity_text') + + def geolocations = geoResult.locations + if (geolocations.size() > 0) { + println " ${geolocations[0].source}-${geolocations[0].country} ${geolocations.size}" + } else { + println " notfound" + return; + } + + // Don't store bad coordinate markers. + if (geolocations[0].latitude == BAD_COORDINATE) { + return; + } + + db.setGeoLocations(node, geolocations as GeoLocation[]) + + ++locationCount + if (locationCount % 1000 == 0) { + println "Committing at ${locationCount}" + db.commit() + } + } + + + // Regexps for cleanup() + final static LEADING_DASHES = /^[- ]/ + + + def cleanup(place) { + return place.replaceFirst(LEADING_DASHES, '') + + } + + def geocodeOne(node) { + def place = node.getProperty('entity_text') + //println "geocoding $place" + + // First see if it's a specified lat/logn. + def geolocations = parse_lat_long(place) + if (geolocations != null) { + // If size = 0, the string parsed OK, but no geolocations found + if (geolocations.size() != 0) { + queueGeoLocations(node, geolocations) + return + } + } + + // For lookup, clean up junk characters, etc. + place = cleanup(place) + if (place == '') { + queueGeoLocations(node, []) + return + } + + // Look in OSM + if (osmFetcher != null) { + geolocations = osmFetcher.getGeoLocations(place) + if (geolocations.size() != 0) { + queueGeoLocations(node, geolocations) + return + } + } + + + + + } + + + void geocode(boolean corefOnly=false) { + + + def location_nodes = corefOnly ? + db.getEntitiesCreatedBy("across_doc_location_coref").toList().sort{it['entity_text']} : + db.getEntitiesOfType(E_LOCATION).toList().sort{it['entity_text']} + + for (node in location_nodes) { + final finalNode = node + // println "queuing ${node.getProperty('entity_text')}" + geocodeOne(finalNode) + } + + + + // Store all the results. + + geoResultList.each { storeGeoLocations(it) } + + //db.rollback() + db.commit() + println "Processed: ${geoResultList.size()} locations" + } +} diff --git a/src/GraphDBToH2.groovy b/src/GraphDBToH2.groovy new file mode 100644 index 0000000..a251961 --- /dev/null +++ b/src/GraphDBToH2.groovy @@ -0,0 +1,168 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB +import edu.mit.ll.vizlincdb.relational.VizLincRDB + +import groovy.sql.Sql +import java.sql.SQLException +import org.apache.commons.io.FileUtils +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +class GraphDBToH2 { + + // Fields + def db // h2 db object + VizLincDB gdb + + + GraphDBToH2(graphdb) { + gdb = graphdb + } + + + // For running standalone. + static main(args) { + if (args.size() != 2) { + println "Usage: GraphDBToH2.groovy " + System.exit(1) + } + + def graphdb = new VizLincDB(args[0]) + GraphDBToH2 graphDBToH2 = new GraphDBToH2(graphdb) + graphDBToH2.convert(args[1]) + println "shutting down graph db" + graphdb.shutdown() + } + + + def convert(h2dbFilename) { + def h2dbFile = new File(h2dbFilename) + FileUtils.deleteQuietly(h2dbFile) + h2dbFile.mkdir() + db = Sql.newInstance("jdbc:h2:${h2dbFile}/data;LOG=0;CACHE_SIZE=0;LOCK_MODE=0;UNDO_LOG=0", 'sa', '', 'org.h2.Driver') + + // Create all the tables, indices, etc. + db.execute(VizLincRDB.H2_SCHEMA) + + convertEntities() + convertDocuments() + convertEdges() + convertMentions() + + println "closing h2 db..." + db.close() + println "... closed" + } + + + private void convertMentions() { + gdb.getMentions().eachWithIndex { mention, count -> + if (count % 1000 == 0) { + println "mention $count" + db.commit(); + } + def mention_id = mention.getId() + def document_id = VizLincDB.makeList(mention.getVertices(Direction.IN, L_DOCUMENT_TO_MENTION))[0].getId() + def entities = VizLincDB.makeList(mention.getVertices(Direction.OUT, L_MENTION_TO_ENTITY)) // should be length 0 or 1 + def entity_id = (entities.size() == 1) ? entities[0].getId() : null + def type = mention.getProperty(P_MENTION_TYPE) + def text = mention.getProperty(P_MENTION_TEXT) + def index = mention.getProperty(P_MENTION_INDEX) + def global_id = mention.getProperty(P_MENTION_GLOBAL_ID) + def text_start = mention.getProperty(P_MENTION_TEXT_START) + def text_stop = mention.getProperty(P_MENTION_TEXT_STOP) + + try { + // Triple-quoted Groovy Strings actually generate a prepared statement. + db.execute("""INSERT INTO mention (mention_id, document_id, entity_id, type, text, index, global_id, text_start, text_stop) + VALUES(${mention_id}, ${document_id}, ${entity_id}, ${type}, ${text}, ${index}, ${global_id}, ${text_start}, ${text_stop} )""") + } catch (SQLException e) { + e.printStackTrace() + println("""VALUES: ${mention_id}, ${document_id}, ${entity_id}, ${type}, ${text}, ${index}, ${global_id}, ${text_start}, ${text_stop}""") + } + } + } + + + private void convertEdges() { + gdb.getEdges(L_DOCUMENT_TO_ENTITY).eachWithIndex { edge, count -> + if (count % 1000 == 0) { + println "document->entity $count" + db.commit(); + } + def doc = edge.getVertex(Direction.OUT) + def entity = edge.getVertex(Direction.IN) + assert doc.getProperty(P_NODE_TYPE) == NODE_TYPE_DOCUMENT + assert entity.getProperty(P_NODE_TYPE) == NODE_TYPE_ENTITY + def document_id = doc.getId() + def entity_id = entity.getId() + def num_mentions = edge.getProperty(P_NUM_MENTIONS) + // println ("doc ${document_id} entity ${entity_id} num_mentions ${num_mentions}") + // Use MERGE because duplicate edges will cause key errors. + db.execute("""MERGE INTO document_entity (document_id, entity_id, num_mentions) + VALUES(${document_id}, ${entity_id}, ${num_mentions})""") + } + } + + + private void convertDocuments() { + gdb.getDocuments().eachWithIndex { doc, count -> + if (count % 1000 == 0) { + println "adding document $count" + db.commit(); + } + def document_id = doc.getId() + def name = doc.getProperty(P_DOCUMENT_NAME) + def path = doc.getProperty(P_DOCUMENT_PATH) + def text = doc.getProperty(P_DOCUMENT_TEXT) + + db.execute("""INSERT INTO document (document_id, name, path, text) + VALUES(${document_id}, ${name}, ${path}, ${text})""") + } + } + + + private void convertEntities() { + gdb.getEntities().eachWithIndex { entity, count -> + if (count % 1000 == 0) { + println "entity $count" + db.commit(); + } + + def entity_id = entity.getId() + def type = entity.getProperty(P_ENTITY_TYPE) + def text = entity.getProperty(P_ENTITY_TEXT) + def created_by = entity.getProperty(P_CREATED_BY) + def num_documents = entity.getProperty(P_NUM_DOCUMENTS) + def num_mentions = entity.getProperty(P_NUM_MENTIONS) + def geolocations = gdb.getGeoLocations(entity) + + db.execute("""INSERT INTO entity (entity_id, type, text, created_by, num_documents, num_mentions) + VALUES(${entity_id}, ${type}, ${text}, ${created_by}, ${num_documents}, ${num_mentions})""") + + if (geolocations != null) { + geolocations.eachWithIndex { loc, rank -> + def bbox = loc.boundingBox + if (bbox != null && bbox.isValid()) { + db.execute("""INSERT INTO geolocation (entity_id, rank, latitude, longitude, + latitude_south, latitude_north, longitude_west, longitude_east, + name, osm_type, nga_designation, country, source) + VALUES(${entity_id}, ${rank}, ${loc.latitude}, ${loc.longitude}, + ${bbox.latitudeSouth}, ${bbox.latitudeNorth}, ${bbox.longitudeWest}, ${bbox.longitudeEast}, + ${loc.name}, ${loc.osmType}, ${loc.ngaDesignation}, ${loc.country}, ${loc.source})""") + } else { + db.execute("""INSERT INTO geolocation (entity_id, rank, latitude, longitude, + name, osm_type, nga_designation, country, source) + VALUES(${entity_id}, ${rank}, ${loc.latitude}, ${loc.longitude}, + ${loc.name}, ${loc.osmType}, ${loc.ngaDesignation}, ${loc.country}, ${loc.source})""") + + } + } + } + } + } + + +} \ No newline at end of file diff --git a/src/Ingester.groovy b/src/Ingester.groovy new file mode 100644 index 0000000..dc8c17d --- /dev/null +++ b/src/Ingester.groovy @@ -0,0 +1,300 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.util.ElapsedTime +import edu.mit.ll.vizlincdb.document.VizLincDocumentIndexer +import edu.mit.ll.vizlincdb.graph.VizLincDB +import edu.mit.ll.vizlincdb.io.VizLincGraphPopulator + +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +import groovy.io.FileType +import java.nio.file.Path +import org.apache.commons.io.FilenameUtils + + + + +// Complete ingestion pipeline: + +class Ingester { + + static final DEFAULT_DATA_DIR = 'data' + static final DEFAULT_MODEL_NAME = 'ner-model.ser.gz' + + static final TXT_EXTENSION = '.txt' + static final NER_EXTENSION = '.ner.csv' + + Ingester() { + } + + enum Step { + EXTRACT_TEXT('Extract text'), + FIND_NAMED_ENTITIES('Find named entities'), + POPULATE_GRAPH_DB('Populate graph database with documents and entity mentions'), + PROCESS_PEOPLE('Process people (including social network)'), + PROCESS_ORGANIZATIONS('Process organizations'), + PROCESS_LOCATIONS('Process locations'), + GEOCODE('Geocode locations'), + PRECOMPUTE_COUNTS('Precompute counts'), + INDEX_DOCS('Index documents'), + CONVERT_TO_H2('Convert to H2 database') + + private final String text + Step(String text) { + this.text = text + } + + public String text() { + return text + } + + public String toString() { + return text + } + } + + + // For running standalone. + static main(args) { + Gremlin.load() + def cli = new CliBuilder(usage: 'Ingester.groovy [options]') + cli.with { + h(longOpt: 'help', 'show usage') + i(longOpt: 'input-documents-dir', required: true, args: 1, argName: 'dir', + 'original documents dir (.pdf, etc.)') + o(longOpt: 'output-dir', required: true, args: 1, argName: 'dir', + 'dir for final .h2, .lucene, and .graphml files') + w(longOpt: 'work-dir', required: true, args: 1, argName: 'dir', + 'dir for intermediate results; WARNING: existing content may be deleted or overwritten') + m(longOpt: 'model-filename', required: false, args: 1, argName: 'model', + "Stanford NER model filename (default: '/${DEFAULT_MODEL_NAME}')") + b(longOpt: 'basename', required: true, args: 1, argName: 'basename', + 'basename for the .neo4j, .graphml, .h2, and .lucene files and dirs') + d(longOpt: 'data-dir', required: false, args: 1, argName: 'dir', + "dir for model and geocoding data files (default: '${DEFAULT_DATA_DIR}')") + } + + def options = cli.parse(args) + if (!options) return + if (!options || options.h) { + cli.usage() + return + } + + // Default for data-dir is 'data'. + def dataDir = new File(options.'data-dir' ? options.'data-dir' : DEFAULT_DATA_DIR) + def modelFile = options.'model-filename' ? new File(options.'model-filename') : new File(DEFAULT_DATA_DIR, DEFAULT_MODEL_NAME) + + Ingester ingester = new Ingester() + ingester.ingest(options.'basename', + new File(options.'input-documents-dir'), + new File(options.'output-dir'), + new File(options.'work-dir'), + dataDir, + modelFile) + } + + + // Full ingestion pipeline + void ingest(String basename, File originalDocumentsDir, File outputDir, File workDir, File dataDir, File nerModelFilename, Closure startStep = { step -> }, Closure stopStep = { step -> }) { + + if (!outputDir.exists()) { + outputDir.mkdirs() + } + + if (!workDir.exists()) { + workDir.mkdirs() + } + + def totalElapsedTime = new ElapsedTime() + + // Convenience function. + def runStep = { step, code -> + def elapsedTime = new ElapsedTime() + println "*** ${step.text()}" + startStep(step) + code() + stopStep(step) + elapsedTime.done("**** finished ${step.text()} in (secs)") + } + + def txtDir = new File(workDir, basename + "${TXT_EXTENSION}-files") + runStep(Step.EXTRACT_TEXT, { + extractTextFromDocuments(originalDocumentsDir, txtDir, filesOnPath(originalDocumentsDir, ''), TXT_EXTENSION) + }) + + def nerDir = new File(workDir, basename + "${NER_EXTENSION}-files") + runStep(Step.FIND_NAMED_ENTITIES, { + markNamedEntities(txtDir, nerDir, filesOnPath(txtDir, TXT_EXTENSION), nerModelFilename, NER_EXTENSION) + }) + + def dbPath = new File(workDir, basename + '.neo4j') + if (dbPath.exists()) { + // Delete directory tree, or file if it was just a file. + if (!dbPath.deleteDir()) { + dbPath.delete() + } + } + + // VizLincGraphPopulator takes a dbPath, not a db instance, because it opens the db as a BatchGraph database for speed. + runStep(Step.POPULATE_GRAPH_DB, { + def graphPopulator = new VizLincGraphPopulator(dbPath.getPath()) + try { + addDocumentsAndMentionsToGraphDB(graphPopulator, txtDir, filesOnPath(txtDir, TXT_EXTENSION), + TXT_EXTENSION, nerDir, NER_EXTENSION) + } finally { + // Shutdown even on error (e.g. file not found). If not the database can be left in a bad state. + graphPopulator.shutdown() + } + }) + + // Now open the database as a transactional graph db. + def db = new VizLincDB(dbPath) + try { + runStep(Step.PROCESS_PEOPLE, { + processPeople(db, new File(outputDir, basename + '.graphml')) + }) + + runStep(Step.PROCESS_ORGANIZATIONS, { + processOrganizations(db) + }) + + runStep(Step.PROCESS_LOCATIONS, { + processLocations(db) + }) + + runStep(Step.GEOCODE, { + geocodeLocations(db) + }) + + runStep(Step.PRECOMPUTE_COUNTS, { + computeCounts(db) + }) + + runStep(Step.INDEX_DOCS, { + indexDocuments(db, new File(outputDir, basename + '.lucene')) + }) + + // When all done with db: + runStep(Step.CONVERT_TO_H2, { + convertToH2(db, new File(outputDir, basename + '.h2')) + println "Committing neo4j db ..." + db.commit() + println "... finished committing" + }) + + } finally { + println "Shutting down neo4j db ..." + db.shutdown() + println "... finished shutting down" + totalElapsedTime.done("**** Entire ingestion finished in (secs)") + } + } + + + List filesOnPath(File path, String extension) { + def filenames = [] + Path rootPathObj = path.toPath() + path.eachFileRecurse(FileType.FILES) { + if (it.name.endsWith(extension)) { + filenames.add(rootPathObj.relativize(it.toPath()).toString()) + } + } + + return filenames.sort() + } + + + void extractTextFromDocuments(File originalDocumentsDir, File txtDir, List documentFileNames, String txtFilesExtension) { + if (txtDir.exists()) { + println "Deleting existing $txtDir" + txtDir.deleteDir() + } + txtDir.mkdirs(); + + TextExtractor textExtractor = new TextExtractor() + for (docFileName in documentFileNames) { + println "Extracting text from: ${docFileName}" + def outputFile = new File(txtDir, docFileName + txtFilesExtension) + outputFile.getParentFile().mkdirs() // Create any necessary subdirs. + textExtractor.extractText(new File(originalDocumentsDir, docFileName), outputFile) + } + } + + + void markNamedEntities(File txtDir, File nerDir, List txtFileNames, File modelFile, String nerFilesExtension) { + if (nerDir.exists()) { + println "Deleting existing $nerDir" + nerDir.deleteDir() + } + nerDir.mkdirs(); + + EntityExtractor entityExtractor = new EntityExtractor(modelFile) + for (txtFileName in txtFileNames) { + println "finding named entities in: ${txtFileName}" + def outputFile = new File(nerDir, txtFileName + nerFilesExtension) + outputFile.getParentFile().mkdirs() // Create any necessary subdirs. + entityExtractor.extractEntitiesToCSV(new File(txtDir, txtFileName), outputFile) + } + } + + + void addDocumentsAndMentionsToGraphDB(VizLincGraphPopulator populator, File txtDir, List txtFileNames, String txtExtension, File nerDir, String nerExtension) { + for (txtFileName in txtFileNames) { + String nerFileName = txtFileName + nerExtension + println "populating graph with document: ${txtFileName} and mentions: ${nerFileName}" + File txtFile = new File(txtDir, txtFileName) + populator.ingestDocumentAndMentions(new File(txtDir, txtFileName), txtFileName, new File(nerDir, nerFileName)) + } + } + + + void processPeople(VizLincDB db, File graphmlFilename) { + // Find person coreferences within and across docs. + (new WithinDocPerCoref(db)).findCoreferences() + (new AcrossDocPerCoref(db)).findCoreferences() + + // Compute the person social network, based on co-occurences across docs. + (new SocialNetworkDCO(db)).computeSocialNetwork() + + // Convert the social network to a graph, and filter it somewhat. + (new SNGephiGraphML(db)).generate(graphmlFilename.getPath()) + } + + void processOrganizations(VizLincDB db) { + // Find organization coreferences within and across docs. + (new WithinDocOrgCoref(db)).findCoreferences() + (new AcrossDocOrgCoref(db)).findCoreferences() + } + + void processLocations(VizLincDB db) { + // Find location coreferences within and across docs. + (new WithinDocLocCoref(db)).findCoreferences() + (new AcrossDocLocCoref(db)).findCoreferences() + } + + void geocodeLocations(VizLincDB db) { + // Try to geocode all locations. + println("geocode locations") + (new Geocoder(db)).geocode() + } + + void computeCounts(VizLincDB db) { + // Count num mentions and put on edges. + (new EdgeMentionsCounter(db)).countMentions() + } + + void indexDocuments(VizLincDB db, File indexPath) { + // Generate a lucene index from the documents in the database. + VizLincDocumentIndexer indexer = new VizLincDocumentIndexer(indexPath.getPath()) + indexer.indexDocuments(db) + indexer.close() + } + + void convertToH2(VizLincDB db, File h2dbFile) { + // Convert the final graph db to an H2 db. + (new GraphDBToH2(db)).convert(h2dbFile.getPath()) + } +} diff --git a/src/IngesterGUI.groovy b/src/IngesterGUI.groovy new file mode 100644 index 0000000..d2644f9 --- /dev/null +++ b/src/IngesterGUI.groovy @@ -0,0 +1,344 @@ +// GUI interface for Ingester + +import com.tinkerpop.gremlin.groovy.* + +import groovy.ui.SystemOutputInterceptor +import groovy.swing.SwingBuilder +import java.awt.Color +import java.awt.Dimension +import javax.swing.BorderFactory +import javax.swing.JComponent +import javax.swing.JFileChooser +import javax.swing.JFrame +import javax.swing.JLabel +import javax.swing.JOptionPane +import javax.swing.ScrollPaneConstants +import javax.swing.text.DefaultCaret +import javax.swing.UIManager + +class IngesterGUI { + + static final DEFAULT_DATA_DIR = 'data' + + // Fields + File dataDir + SwingBuilder swing + def stepList + def resultsTextArea + def logTextArea + def logScrollPane + + IngesterGUI(File dataDir) { + this.dataDir = dataDir + } + + static final DISALLOWED_BASENAME_CHARS = '\\/:*?"<>|\'' + + def MODEL_FILENAME = 'ner-model.ser.gz' + + // Fields + def uiFrame + def inputFolderChooser + def outputFolderChooser + + static main(args) { + Gremlin.load() + def cli = new CliBuilder(usage: 'IngesterGUI.groovy [options]') + cli.with { + h(longOpt: 'help', 'show usage') + d(longOpt: 'data-dir', required: false, args: 1, argName: 'dir', + "dir for model and geocoding data files (default: '${DEFAULT_DATA_DIR}')") + } + + def options = cli.parse(args) + if (!options) return + if (!options || options.h) { + cli.usage() + return + } + + def dataDir = new File(options.'data-dir' ? options.'data-dir' : DEFAULT_DATA_DIR) + IngesterGUI ingesterGUI = new IngesterGUI(dataDir) + // Ingestion will run when the user presses the button + ingesterGUI.setupGUI() + } + + + // Arguments have already been validated. + void ingest(String basename, File inputFolder, File outputFolder, Closure startStep, Closure stopStep) { + + def workDir = new File(outputFolder, 'work') + def modelFile = new File(dataDir, MODEL_FILENAME) + + // Create output folder now so we can create the log file. + if (!outputFolder.exists()) { + outputFolder.mkdirs() + } + + // Capture stdout and stderr and write them to a log file in the output dir. + def logFile = new File(outputFolder, basename + '.log') + def logFilePrintStream = new PrintStream(logFile, "UTF-8") + def lastLogFlushMillis = System.currentTimeMillis() + + setResult('') + appendLineToResult("Logging detailed output below and to: ${logFile}.") + + // Clear any existing logging. + logTextArea.setText('') + + final CHARS_TO_BUFFER = 1024 + final LOG_INTERVAL_MILLIS = 3000 + def logBuffer = new StringBuilder(CHARS_TO_BUFFER) + def flushLogBuffer = { + // Scrolling happens automatically due to DefaultCaret update policy. + logTextArea.append(logBuffer.toString()) + logBuffer.setLength(0) + } + + def writeToLogTextArea = { msg -> + logBuffer.append(msg) + // Flush if the buffer is big enough or it's been a while since the last flush. + def currentTimeMillis = System.currentTimeMillis() + if (logBuffer.length() > CHARS_TO_BUFFER || (currentTimeMillis - lastLogFlushMillis) > LOG_INTERVAL_MILLIS) { + flushLogBuffer() + lastLogFlushMillis = currentTimeMillis + } + return true; // Also write to stdout/stderr + } + + PrintStream stdout = System.out + PrintStream stderr = System.err + + System.setOut(logFilePrintStream) + System.setErr(logFilePrintStream) + + def stdoutInterceptor = new SystemOutputInterceptor(writeToLogTextArea, true) + def stderrInterceptor = new SystemOutputInterceptor(writeToLogTextArea, false) + + stdoutInterceptor.start() + stderrInterceptor.start() + + def startTime = System.nanoTime() + + Ingester ingester = new Ingester() + ingester.ingest(basename, + inputFolder, + outputFolder, + workDir, + dataDir, + modelFile, + startStep, + stopStep) + + flushLogBuffer() + + stdoutInterceptor.stop() + stderrInterceptor.stop() + + System.setOut(stdout) + System.setErr(stderr) + + logFilePrintStream.close() + + def elapsedSeconds = (System.nanoTime() - startTime) / 1.0e9 + + appendLineToResult("Done. Results are in ${outputFolder}") + appendLineToResult("Time: ${String.format("%.2f", elapsedSeconds)} seconds") + + // Oddly, this needs to be done twice. Maybe a repaint issue. + flushLogBuffer() + + } + + + void setupGUI() { + + swing = new SwingBuilder() + final FOLDER_NAME_WIDTH = 40 + + // Input folder UI + def inputFolderTextField = swing.textField(columns: FOLDER_NAME_WIDTH) + // Don't let it shrink to nothing. + inputFolderTextField.setMinimumSize(inputFolderTextField.getPreferredSize()); + def chooseInputFolderButton = swing.button('Choose ...') + inputFolderChooser = swing.fileChooser(fileSelectionMode: JFileChooser.DIRECTORIES_ONLY) + chooseInputFolderButton.actionPerformed = { + if (inputFolderChooser.showOpenDialog() != JFileChooser.APPROVE_OPTION) return + inputFolderTextField.text = inputFolderChooser.selectedFile + } + + // Output folder UI + def outputFolderTextField = swing.textField(columns: FOLDER_NAME_WIDTH) + outputFolderTextField.setMinimumSize(outputFolderTextField.getPreferredSize()); + def chooseOutputFolderButton = swing.button('Choose ...') + outputFolderChooser = swing.fileChooser(fileSelectionMode: JFileChooser.DIRECTORIES_ONLY) + chooseOutputFolderButton.actionPerformed = { + if (outputFolderChooser.showOpenDialog() != JFileChooser.APPROVE_OPTION) return + outputFolderTextField.text = outputFolderChooser.selectedFile + } + + // Basename UI + def basenameTextField = swing.textField(columns: FOLDER_NAME_WIDTH) + basenameTextField.setMinimumSize(basenameTextField.getPreferredSize()); + + def startIngestionButton = swing.button('Start Ingestion (will take minutes or hours)', + alignmentX: JComponent.CENTER_ALIGNMENT) + + startIngestionButton.actionPerformed = { + + // Input folder validation. + if (inputFolderTextField.text.isEmpty()) { + JOptionPane.showMessageDialog(uiFrame, 'Please specify an input folder.') + return + } + + def inputFolder = new File(inputFolderTextField.text) + if (!inputFolder.exists()) { + JOptionPane.showMessageDialog(uiFrame, 'The input folder you specified does not exist.') + return + } + if (!inputFolder.isDirectory()) { + JOptionPane.showMessageDialog(uiFrame, 'The input pathname you specified is not a folder.') + return + } + + // Output folder validation + if (outputFolderTextField.text.isEmpty()) { + JOptionPane.showMessageDialog(uiFrame, 'Please specify an output folder.') + return + } + + def outputFolder = new File(outputFolderTextField.text) + if (outputFolder.exists() && !outputFolder.isDirectory()) { + JOptionPane.showMessageDialog(uiFrame, 'The output pathname you specified is not a folder.') + return + } + + // basename validation. + if (basenameTextField.text.isEmpty()) { + JOptionPane.showMessageDialog(uiFrame, 'Please specify an output files prefix.') + return + } + + if (basenameTextField.text =~ /[$DISALLOWED_BASENAME_CHARS]/) { + JOptionPane.showMessageDialog(uiFrame, "Please do not use these characters in the output files prefix: $DISALLOWED_BASENAME_CHARS.") + return + } + + swing.doOutside { + try { + ingest(basenameTextField.text, + inputFolder, + outputFolder, + { step -> swing.edt { highlightStep(step) } }, + { step -> swing.edt { unhighlightStep(step) } }) + } catch (Throwable t) { + t.printStackTrace() + appendLineToResult("ERROR: Exception thrown. Check log file for details.\n") + } + } + } + + uiFrame = swing.frame(title: 'VizLinc Document Ingester 1.0', + defaultCloseOperation:JFrame.EXIT_ON_CLOSE) { + vbox { + tableLayout(cellpadding: 10) { + + tr { + td { label('Input folder
(contains original documents:
.pdf, .txt, .doc, .docx, etc.)') } + td { widget(inputFolderTextField) } + td { widget(chooseInputFolderButton) } + } + + tr { + td { label('Output folder
(.h2, .lucene, .graphml, + a work folder)') } + td { widget(outputFolderTextField) } + td { widget(chooseOutputFolderButton) } + } + + tr { + td { label('Output files prefix') } + td { widget(basenameTextField) } + } + } + + vstrut(20) + hbox { + hstrut(20) + vbox { + widget(startIngestionButton) + vstrut(20) + resultsTextArea = swing.textArea( + '', + alignmentX: JLabel.CENTER_ALIGNMENT, + rows: 4, + editable: false, cursor: null, + wrapStyleWord: true, lineWrap: false, + border: BorderFactory.createLoweredBevelBorder()) + vstrut(20) + // We should just be able to return to exit, but Groovy 1.8.2 uses an ExecutorService for + // SwingBuilder.doOutside, which is never shutdown. So we have to force an exit. This is fixed in Groovy + // 2.1. See https://jira.codehaus.org/browse/GROOVY-5074. + button('Exit', alignmentX: JLabel.CENTER_ALIGNMENT).actionPerformed = { swing.dispose() ; System.exit(0) } + } + hstrut(width: 20) + vbox { + panel( border: BorderFactory.createTitledBorder('Ingestion Progress'), opaque: false) { + stepList = list(listData: Ingester.Step.values(), opaque: false) + // DOESN'T WORK; selections no longer get highlighted // stepList.getCellRenderer().setOpaque(false) + // Remove mouse listeners instead of disabling, so that the text color doesn't change. + stepList.getMouseListeners().each { stepList.removeMouseListener(it) } + } + } + hstrut(20) + } + vstrut(20) + + + // large logging text area + panel(border: BorderFactory.createCompoundBorder( + BorderFactory.createEmptyBorder(10, 10, 10, 10), + BorderFactory.createTitledBorder('Detailed Ingestion Log'))) { + borderLayout() + logScrollPane = scrollPane(verticalScrollBarPolicy: ScrollPaneConstants.VERTICAL_SCROLLBAR_ALWAYS, + horizontalScrollBarPolicy: ScrollPaneConstants.HORIZONTAL_SCROLLBAR_ALWAYS) { + logTextArea = textArea( + '', + alignmentX: JLabel.CENTER_ALIGNMENT, + rows: 24, columns: 80, + editable: false, + cursor: null) + // Scroll to bottom automatically. + logTextArea.getCaret().setUpdatePolicy(DefaultCaret.ALWAYS_UPDATE) + } + } + } + } + + uiFrame.pack() + uiFrame.show() + } + + + def appendLineToResult(text) { + swing.edt { + resultsTextArea.append(text) + resultsTextArea.append('\n') + } + } + + def setResult(text) { + swing.edt { + resultsTextArea.setText(text) + } + } + + def highlightStep(step) { + stepList.setSelectedValue(step, true /* should scroll */) + } + + def unhighlightStep(step) { + stepList.clearSelection() + } + +} diff --git a/src/NodeAndText.groovy b/src/NodeAndText.groovy new file mode 100644 index 0000000..d8edf61 --- /dev/null +++ b/src/NodeAndText.groovy @@ -0,0 +1,17 @@ +import com.tinkerpop.blueprints.* + +class NodeAndText { + Vertex node + String text + + NodeAndText(Vertex node, String text) { + this.node = node + this.text = text + } + + String toString() { + return "(node=${node}, text=$text)" + } + +} + diff --git a/src/NormalizedDate.groovy b/src/NormalizedDate.groovy new file mode 100644 index 0000000..b7fac29 --- /dev/null +++ b/src/NormalizedDate.groovy @@ -0,0 +1,111 @@ +public class NormalizedDate { + + private String day; + private String month; + private String year; + private static final MONTH_LONG = ["enero", "febrero", "marzo", "abril", "mayo", "junio", + "julio", "agosto", "septiembre", "octubre", "noviembre", "diciembre"] + private static final MONTH_SHORT = ["ene", "feb", "mar", "abr", "may", "jun", "jul", "ago", "sep", "oct", "nov", "dic"] + + NormalizedDate(String day, String month, String year) { + //Normalize each field + this.day = resolveDay(day); + if (this.day == null) { + throw new IllegalArgumentException(day + " is not a valid day."); + } + this.month = resolveMonth(month); + if (this.month == null) { + throw new IllegalArgumentException(month + "is not a valid month"); + } + this.year = resolveYear(year); + if (this.year == null) { + throw new IllegalArgumentException(year + "is not a valid year"); + } + } + + /* + * YYYY/MM/DD format + */ + @Override + public String toString() { + return year + "/" + month + "/" + day; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof NormalizedDate)) { + return false; + } + + NormalizedDate n2 = (NormalizedDate) obj; + return toString().equals(n2.toString()); + } + + private String resolveDay(String day) { + int dayInt = Integer.parseInt(day); + if (dayInt > 0 && dayInt < 10) { + return "0" + dayInt; + } else if (dayInt >= 10 && dayInt <= 31) { + return String.valueOf(dayInt); + } else { + return null; + } + } + + private String resolveMonth(String month) { + int index = -1; + if (month.length() > 3) { + index = MONTH_LONG.findIndexOf {it == month}; + } else { + index = MONTH_SHORT.findIndexOf {it == month} + } + if (index == -1) { + return null; + } + + String monthNum = String.valueOf(index + 1); + + if (monthNum.length() == 2) { + return monthNum; + } else if (monthNum.length() == 1) { + return "0" + monthNum; + } else { + return null; + } + + } + + private String resolveYear(String year) { + year = year.replaceAll("[^0-9]", ""); + if (year.length() == 2) { + return resolve2Year(year); + } else if (year.length() == 4) { + return resolve4Year(year); + } else { + return null; + } + } + + private String resolve2Year(String year) { + int addYears = Integer.parseInt(year); + if (addYears >= 0 && addYears <= 12) { + return String.valueOf(2000 + addYears); + } else if (addYears > 12) { + return String.valueOf(1900 + addYears); + } else { + return null; + } + } + + private String resolve4Year(String year) { + int yearInt = Integer.parseInt(year); + if(yearInt >=1900 && yearInt <=2012) + { + return year; + } + else + { + return null; + } + } +} diff --git a/src/OSMFetcher.groovy b/src/OSMFetcher.groovy new file mode 100644 index 0000000..91abd56 --- /dev/null +++ b/src/OSMFetcher.groovy @@ -0,0 +1,38 @@ +import edu.mit.ll.vizlincdb.geo.GeoBoundingBox +import edu.mit.ll.vizlincdb.geo.GeoLocation + +class OSMFetcher { + + + static String baseURL = 'http://open.mapquestapi.com/nominatim/v1/search.php' + + def getGeoLocations(String location) { + URL mapquest = new URL("${baseURL}?format=xml&q=${URLEncoder.encode(location, 'UTF-8')}&addressdetails=1&limit=1") + return getGeoLocationsFromURL(new URL("${baseURL}?format=xml&q=${URLEncoder.encode(location, 'UTF-8')}&addressdetails=1&limit=1")) + } + + def getGeoLocationsInCountry(String countryCode, String location) { + return getGeoLocationsFromURL(new URL("${baseURL}&addressdetails=1&format=xml&bounded=0&osm_type=N&q=${URLEncoder.encode(location, 'UTF-8')}")) + } + + def getGeoLocationsFromURL(URL url) { + try { + def xml = url.getText() + def path = new XmlSlurper().parseText(xml) + assert path.name() == 'searchresults' + return path['place'].collect { place -> + def (latS, latN, lonW, lonE) = place['@boundingbox'].text().tokenize(',').collect{it.toDouble()} + GeoLocation.OSMGeoLocation( + place['@lat'].toDouble(),place['@lon'].toDouble(), + new GeoBoundingBox(latS, latN, lonW, lonE), + place['@display_name'].text(), + place['@type'].text(), + place['country_code'].text()) + } + } catch (IOException e) { + println("exception") + println(e) + return [] + } + } +} \ No newline at end of file diff --git a/src/SNGephiGraphML.groovy b/src/SNGephiGraphML.groovy new file mode 100644 index 0000000..c1b8a54 --- /dev/null +++ b/src/SNGephiGraphML.groovy @@ -0,0 +1,104 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB + +import com.tinkerpop.blueprints.impls.tg.* +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +// Create a Gephi GRAPHML file -- might create isolates +// Adapted from sn_gephi_gdf +class SNGephiGraphML { + + static NODE_MIN_DOC_REF = 2 + static EDGE_MIN_DOC_REF = 2 + + // Fields + VizLincDB db + + SNGephiGraphML(graphdb) { + db = graphdb + } + + static main(args) { + Gremlin.load() + if (args.size() < 2) { + println "Usage: SNGephiGraphML.groovy " + println "Retrieve and filter the social network from the database and save to the GRAPHML file format." + println "A person must be mentioned in at least $NODE_MIN_DOC_REF documents." + println "Two people must be mentioned together in at least $EDGE_MIN_DOC_REF documents." + System.exit(1) + } + + def graphdb = new VizLincDB(args[0]) + SNGephiGraphML snGephiGraphML = new SNGephiGraphML(graphdb) + snGephiGraphML.generate(args[1]) + graphdb.shutdown() + } + + void generate(graphmlFilename) { + def outfile = new FileWriter(graphmlFilename) + outfile.write('''\ + + + + + + +''') + + // Query for frequent entity nodes + def per_nodes = db.graph.V(P_CREATED_BY,'across_doc_person_coref').filter{it.getProperty('num_docs') >= NODE_MIN_DOC_REF}.sort{it.getProperty('entity_text')} + def per_final = new HashSet() + def node_count = 0 + for (per in per_nodes) { + node_count += 1 + def per_name = per['entity_text'] + // if (!(per_name ==~ /[A-Z\s\.]+/)) { + // println "Unusual entity string, skipping: |" + per_name + "|" + // continue + // } + def per_id = per.getId() + per_final.add(per_id) + outfile.write("""\ + ${per_name}${per_id} +""") + } + + // Edges + def edge_iter = db.graph.E.has('label', L_RELATED_ENTITY)._().filter{it.getProperty('num_docs') >= EDGE_MIN_DOC_REF} + def total_edge_count = 0 + def included_edge_count = 0 + for (edge in edge_iter) { + if ((total_edge_count % 10000)==0) { + println "At edge: " + total_edge_count + } + total_edge_count += 1 + + def edge_id = edge.getId() + def p1 = edge.inV.next() + def p2 = edge.outV.next() + def p1_id = p1.getId() + def p2_id = p2.getId() + if (per_final.contains(p1_id) && per_final.contains(p2_id)) { + included_edge_count += 1 + outfile.write("""\ + ${edge_id} +""") + } + } + + outfile.write('''\ + + +''') + + println("Total nodes: ${node_count}") + println("Total edges: ${total_edge_count}") + println("Included edges: ${included_edge_count}") + + outfile.close() + } + +} \ No newline at end of file diff --git a/src/SNNodeId.groovy b/src/SNNodeId.groovy new file mode 100644 index 0000000..476e438 --- /dev/null +++ b/src/SNNodeId.groovy @@ -0,0 +1,59 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB + +import com.tinkerpop.blueprints.impls.tg.* +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +// +// Get node Ids for nodes in Graph DB +// +class SNNodeId { + + static NODE_MIN_DOC_REF = 1 + + // Fields + VizLincDB db + + SNNodeId(graphdb) { + db = graphdb + } + + static main(args) { + Gremlin.load() + if (args.size() < 2) { + println "Usage: SNNodeId.groovy " + println "Get the node Ids for all nodes in the graph DB." + System.exit(1) + } + + def graphdb = new VizLincDB(args[0]) + SNNodeId snNodeId = new SNNodeId(graphdb) + snNodeId.saveNodeIds(args[1]) + graphdb.shutdown() + } + + void saveNodeIds(outFile) { + def outfile = new FileWriter(outFile) + + // Query for frequent entity nodes + def per_nodes = db.graph.V(P_CREATED_BY,'across_doc_person_coref').filter{it.getProperty('num_docs') >= NODE_MIN_DOC_REF}.sort{it.getProperty('entity_text')} + for (per in per_nodes) { + def per_name = per['entity_text'] + def per_id = per.getId() + outfile.write(per_id + "\t" + per_name + "\n") + } + + per_nodes = db.graph.V(P_CREATED_BY,'within_doc_person_coref').filter{it.getProperty('num_docs') >= NODE_MIN_DOC_REF}.sort{it.getProperty('entity_text')} + for (per in per_nodes) { + def per_name = per['entity_text'] + def per_id = per.getId() + outfile.write(per_id + "\t" + per_name + "\n") + } + outfile.close() + + } + +} diff --git a/src/SocialNetworkDCO.groovy b/src/SocialNetworkDCO.groovy new file mode 100644 index 0000000..8533fbc --- /dev/null +++ b/src/SocialNetworkDCO.groovy @@ -0,0 +1,85 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB + +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +// Build a social network using doc co-occurrence + +class SocialNetworkDCO { + + // Fields + VizLincDB db + + SocialNetworkDCO(VizLincDB graphdb) { + db = graphdb + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: SocialNetworkDCO.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + SocialNetworkDCO sn = new SocialNetworkDCO(db) + sn.computeSocialNetwork() + db.shutdown() + } + + // Return the edge that exists between n1 and n2 or null if there isn't one. + def existing_edge (n1, n2) { + def edges = n1.bothE.as('x').bothV.retain([n2]).back('x').toList() + switch (edges.size()) { + case 0: + return null + case 1: + // println "edge: ${n1.getProperty('entity_text')} -- ${n2.getProperty('entity_text')}" + return edges[0] + default: + println "ERROR: multiple edges between $n1 and $n2" + System.exit(2) + } + + } + + + void computeSocialNetwork() { + // Query for docs + def doc_nodes = db.getDocuments() + def i = 0 + def edge_count = 0 + for (doc in doc_nodes) { + // Find persons mentioned in a document + def per_list = doc.outE(L_DOCUMENT_TO_MENTION).inV.outE(L_MENTION_TO_ENTITY).inV.has(P_CREATED_BY, "across_doc_person_coref").unique().toList() + + // Status + println "doc " + i + ", id " + doc.getId() + ", Per list size " + per_list.size() + + // Connect co-occurrences: try all pairs of people. + for (int j=0; j" + System.exit(1) + } + def db = new VizLincDB(args[0]) + WithinDocLocCoref coref = new WithinDocLocCoref(db) + coref.findCoreferences() + db.shutdown() + } + + + void findCoreferences() { + def i = 0 + def DEBUG = false + + def DIST = new Levenstein() + final THRESH = -1.1 + + def doc_nodes = db.getDocuments() + for (node in doc_nodes) { + println "Document node " + i + " : " + node + def location_list = getNormalizedMentions(node, E_LOCATION, JUNK_CHARS_TO_REMOVE_FOR_LOCS) + if (DEBUG) { println "Location list: " + location_list } + println "Number of names: " + location_list.size() + def clusters = combineByExactMatch(location_list) + println "Exact string match, number of clusters: " + clusters.size() + if (DEBUG) { output_clusters(clusters) } + combineAdjacentClose(clusters, DIST, THRESH, true) + println "Combine adjacent, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + combineFirstnameFullname(clusters) + println "Combine first name, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + addEntitiesToGraph(clusters, E_LOCATION, 'within_doc_location_coref') + print "\n\n" + i += 1 + } + db.commit() + } + +} diff --git a/src/WithinDocOrgCoref.groovy b/src/WithinDocOrgCoref.groovy new file mode 100644 index 0000000..5991f37 --- /dev/null +++ b/src/WithinDocOrgCoref.groovy @@ -0,0 +1,63 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB + +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +import com.wcohen.ss.* + +// Within document coref for organizations tags + +class WithinDocOrgCoref extends CorefBase { + + WithinDocOrgCoref(VizLincDB graphdb) { + super(graphdb) + } + + // For running standalone. + static main(args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: WithinDocOrgCoref.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + WithinDocOrgCoref coref = new WithinDocOrgCoref(db) + coref.findCoreferences() + db.shutdown() + } + + + void findCoreferences() { + def i = 0 + def DEBUG = false + + def DIST = new Levenstein() + final THRESH = -1.1 + + def doc_nodes = db.getDocuments() + for (node in doc_nodes) { + println "Document node " + i + " : " + node + def organization_list = getNormalizedMentions(node, E_ORGANIZATION) + if (DEBUG) { println "Organization list: " + organization_list } + println "Number of names: " + organization_list.size() + def clusters = combineByExactMatch(organization_list) + println "Exact string match, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + combineAdjacentClose(clusters, DIST, THRESH, true) + println "Combine adjacent, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + // combineFirstnameFullname(clusters) // Not sure if this is a good thing for orgs + // println "Combine first name, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + addEntitiesToGraph(clusters, E_ORGANIZATION, 'within_doc_organization_coref') + print "\n\n" + i += 1 + } + + db.commit() + } +} + diff --git a/src/WithinDocPerCoref.groovy b/src/WithinDocPerCoref.groovy new file mode 100644 index 0000000..920da2d --- /dev/null +++ b/src/WithinDocPerCoref.groovy @@ -0,0 +1,61 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB + +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +import com.wcohen.ss.* + +// Within document coref for person tags + +class WithinDocPerCoref extends CorefBase { + + WithinDocPerCoref(VizLincDB graphdb) { + super(graphdb) + } + + // For running standalone. + static main(String[] args) { + Gremlin.load() + if (args.size() != 1) { + println "Usage: WithinDocPerCoref.groovy " + System.exit(1) + } + def db = new VizLincDB(args[0]) + WithinDocPerCoref coref = new WithinDocPerCoref(db) + coref.findCoreferences() + db.shutdown() + } + + void findCoreferences() { + def i = 0 + def DEBUG = false + + def DIST = new Levenstein() + final THRESH = -1.1 + + def doc_nodes = db.getDocuments() + for (node in doc_nodes) { + println "Document node " + i + " : " + node + def person_list = getNormalizedMentions(node, E_PERSON) + if (DEBUG) { println "Person list: " + person_list } + println "Number of names: " + person_list.size() + def clusters = combineByExactMatch(person_list) + println "Exact string match, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + combineAdjacentClose(clusters, DIST, THRESH, false) + println "Combine adjacent, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + combineFirstnameFullname(clusters) + println "Combine first name, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + addEntitiesToGraph(clusters, E_PERSON, 'within_doc_person_coref') + print "\n\n" + i += 1 + } + db.commit() + } +} + diff --git a/src/WithinDocSimpleCoref.groovy b/src/WithinDocSimpleCoref.groovy new file mode 100644 index 0000000..56ee014 --- /dev/null +++ b/src/WithinDocSimpleCoref.groovy @@ -0,0 +1,76 @@ +import static edu.mit.ll.vizlincdb.util.VizLincProperties.* +import edu.mit.ll.vizlincdb.graph.VizLincDB + +import com.tinkerpop.blueprints.impls.neo4j.* +import com.tinkerpop.gremlin.groovy.* +import com.tinkerpop.blueprints.* +import com.tinkerpop.gremlin.* + +// Within document coref for given entity type + +class WithinDocSimpleCoref extends CorefBase { + + String entityType // One of E_PERSON, E_LOCATION, etc. + + WithinDocSimpleCoref(String entityType, VizLincDB graphdb) { + super(graphdb) + this.entityType = entityType + } + + // For running standalone. + static main(String[] args) { + Gremlin.load() + if (args.size() != 2) { + println "Usage: WithinDocSimpleCoref.groovy " + System.exit(1) + } + def entityType = args[0] + def db = new VizLincDB(args[1]) + WithinDocSimpleCoref coref = new WithinDocSimpleCoref(entityType, db) + if (entityType==E_LOCATION) + coref.findCoreferences(true) + else + coref.findCoreferences(true) + + db.shutdown() + } + + void findCoreferences(Boolean useGlobalID=false) { + def i = 0 + def DEBUG = false + + if (DEBUG) { + if (useGlobalID) + println "Using global IDs for within doc coref." + else + println "Using exact string match for within doc coref." + } + + def doc_nodes = db.getDocuments() + for (node in doc_nodes) { + println "Document node " + i + " : " + node + + def mentionList = getTwitterNormalizedMentions(node, entityType) + + if (DEBUG) { println "${entityType} list: ${mentionList}" } + println "Number of names: " + mentionList.size() + + def clusters + + if (useGlobalID) { + clusters = combineByGlobalID(mentionList) + addEntitiesToGraphByGlobalID(clusters, entityType, "within_doc_${entityType.toLowerCase()}_coref") + } else { + clusters = combineByExactMatch(mentionList) + addEntitiesToGraph(clusters, entityType, "within_doc_${entityType.toLowerCase()}_coref") + } + println "Exact match, number of clusters: " + clusters.size() + if (DEBUG) { outputClusters(clusters) } + + print "\n\n" + i += 1 + } + db.commit() + } +} + diff --git a/src/XMLToTSVConverter.groovy b/src/XMLToTSVConverter.groovy new file mode 100644 index 0000000..8b6f725 --- /dev/null +++ b/src/XMLToTSVConverter.groovy @@ -0,0 +1,185 @@ +import com.ctc.wstx.stax.WstxInputFactory; +import edu.stanford.nlp.ling.Word; +import edu.stanford.nlp.process.PTBTokenizer; +import edu.stanford.nlp.process.WordTokenFactory; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; +import java.text.ParseException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import org.apache.commons.io.FilenameUtils; +import org.codehaus.stax2.XMLInputFactory2; +import org.codehaus.stax2.XMLStreamReader2; + +/** + * XMLToTSVConverter converts true XML entity-marked transcripts to a Stanford NER .tsv file. + * .tsv is "tab-separated value". + * Example: + * Sam went to entityTags = new HashSet<>(); + String otherTag; + + /** + * An entity parser that handles PERSON, ORGANIZATION, LOCATION, DATE + */ + public XMLToTSVConverter() { + this(["PERSON", "ORGANIZATION", "LOCATION", "DATE"], "O"); + } + + /** + * Handle the entities given in the entityTags list. + * + * entityTags tags + * @param entityTags + * @param otherTag + */ + public XMLToTSVConverter(String[] entityTags, String otherTag) { + Collections.addAll(this.entityTags, entityTags); + this.otherTag = otherTag; + } + + public void convertToTSV(File inputFile, File outputFile) throws IOException, ParseException { + BufferedReader input = new BufferedReader(new InputStreamReader(new FileInputStream(inputFile), "UTF-8")); + BufferedWriter output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile), "UTF-8")); + convertToTSV(input, output); + input.close(); + output.close(); + + } + + public String convertToTSV(String s) throws ParseException, IOException { + BufferedWriter output = new BufferedWriter(new StringWriter()); + convertToTSV(new StringReader(s), output); + return output.toString(); + } + + public void convertToTSV(Reader reader, BufferedWriter writer) throws ParseException, IOException { + try { + // Matches or + // where SOMETHING is PERSON|ORGANIZATION|LOCATION|DATE. + // There may be other entityTags in the document but they are ignored. Phoenix adds these, for instance: + // DOC DOCID DOCTYPE BODY TEXT + + XMLInputFactory2 factory = new WstxInputFactory(); + factory.configureForConvenience(); + // Woodstox + XMLStreamReader2 xmlReader = (XMLStreamReader2) factory.createXMLStreamReader(reader); + + // Don't output until the section. + boolean inTEXT = false; + String currentTag = otherTag; + StringBuilder text = new StringBuilder(); + + while (xmlReader.hasNext()) { + int eventType = xmlReader.next(); + switch (eventType) { + case XMLStreamConstants.START_ELEMENT: + String name = xmlReader.getLocalName(); + if (name.equals("TEXT")) { + // Don't converting until we're actually in the document section. + inTEXT = true; + } else if (inTEXT && entityTags.contains(name)) { + writeTextTokens(writer, text.toString(), currentTag); + text = new StringBuilder(); + currentTag = name; + } + break; + + case XMLStreamConstants.CHARACTERS: + case XMLStreamConstants.CDATA: + // Accumulate text between or outside of tags. + if (inTEXT) { + text.append(xmlReader.getText()); + } + break; + + case XMLStreamConstants.END_ELEMENT: + // If this is a tag we care about, it's closed, and we have all the text inside it. + if (inTEXT && entityTags.contains(xmlReader.getLocalName())) { + writeTextTokens(writer, text.toString(), currentTag); + text = new StringBuilder(); + currentTag = otherTag; + } + break; + + // We don't care about all other states. + default: + break; + } + } + + + } catch (XMLStreamException ex) { + Logger.getLogger(XMLToTSVConverter.class.getName()).log(Level.SEVERE, null, ex); + throw new ParseException(ex.getMessage(), ex.getLocation().getCharacterOffset()); + + } + } + + private void writeTextTokens(BufferedWriter writer, String text, String currentTag) throws IOException { + // System.out.println(text + " --- " + currentTag); + Reader reader = new StringReader(text); + PTBTokenizer tokenizer = new PTBTokenizer(reader, new WordTokenFactory(), /*options*/ ""); + while (tokenizer.hasNext()) { + Word word = tokenizer.next(); + writer.append(word.word()); + writer.append('\t'); + writer.append(currentTag); + writer.newLine(); + } + } + + /** + * @param args args[0]: output directory; args[1..]: input file ... + * @throws java.io.IOException + * @throws java.text.ParseException + */ + public static void main(String[] args) throws IOException, ParseException { + if (args.length < 2) { + System.out.println("Usage: XMLToTSVConverter.groovy ..."); + return; + } + + XMLToTSVConverter converter = new XMLToTSVConverter(); + File outputDir = new File(args[0]); + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + for (int i = 1; i < args.length; i++) { + String inputFilename = args[i]; + String basename = FilenameUtils.getBaseName(inputFilename); + System.out.println("Processing " + inputFilename); + converter.convertToTSV(new File(inputFilename), new File(outputDir, basename + ".tsv")); + + } + } +} diff --git a/vizlinc-ingester-launch4j.xml b/vizlinc-ingester-launch4j.xml new file mode 100644 index 0000000..63f8c02 --- /dev/null +++ b/vizlinc-ingester-launch4j.xml @@ -0,0 +1,33 @@ + + true + gui + + data/splash-ingester.bmp + + dist/lib/vizlinc-ingester.jar + dist/vizlinc-ingester.exe + + -d data + . + normal + http://java.com/download + + false + + + + vizlinc-ingester + VizLinc Ingester + + + IngesterGUI + %EXEDIR%/lib/*.jar + + + + false + 1.7.0_45 + + preferJre + +