消息关闭
    暂无新消息!

求教HADOOP上传文件异常的问题

问题作者 : Lochead2017-05-17发布

代码如下:

import java.io.File;   
import java.io.FileInputStream;   
import java.io.FileNotFoundException;   
import java.io.FileOutputStream;   
import java.io.IOException;   
import java.io.InputStream;   
import java.io.OutputStream;   
import java.net.SocketException;   

import org.apache.commons.net.ftp.FTPClient;   
import org.apache.commons.net.ftp.FTPReply;   


public class FtpClient {
    private String         host;   
    private int            port;   
    private String         username;   
    private String         password;   

    private boolean        binaryTransfer = true;   
    private boolean        passiveMode    = true;   
    private String         encoding       = "UTF-8";   
    private int            clientTimeout  = 3000;   
    private boolean flag=true;
    private FTPClient ftpClient = null;

    public String getHost() {   
        return host;   
    }   

    public void setHost(String host) {   
        this.host = host;   
    }   

    public int getPort() {   
        return port;   
    }   

    public void setPort(int port) {   
        this.port = port;   
    }   

    public String getUsername() {   
        return username;   
    }   

    public void setUsername(String username) {   
        this.username = username;   
    }   

    public String getPassword() {   
        return password;   
    }   

    public void setPassword(String password) {   
        this.password = password;   
    }   

    public boolean isBinaryTransfer() {   
        return binaryTransfer;   
    }   

    public void setBinaryTransfer(boolean binaryTransfer) {   
        this.binaryTransfer = binaryTransfer;   
    }   

    public boolean isPassiveMode() {   
        return passiveMode;   
    }   

    public void setPassiveMode(boolean passiveMode) {   
        this.passiveMode = passiveMode;   
    }   

    public String getEncoding() {   
        return encoding;   
    }   

    public void setEncoding(String encoding) {   
        this.encoding = encoding;   
    }   

    public int getClientTimeout() {   
        return clientTimeout;   
    }   

    public void setClientTimeout(int clientTimeout) {   
        this.clientTimeout = clientTimeout;   
    }   

    public FtpClient(String Host) {
        this.username = "anonymous";
        this.encoding = "utf-8";
        this.binaryTransfer = true;
        this.binaryTransfer = true;
        this.port = 21;
        this.host = Host;
        try {
            this.ftpClient = getFTPClient();
        } catch (Exception e) {
            System.out.println("Create FTPClient error!");
        }
    }

    private FTPClient getFTPClient() throws IOException {   
        FTPClient ftpClient = new FTPClient(); 
        ftpClient.setControlEncoding(encoding);

        connect(ftpClient);
        if (passiveMode) {   
            ftpClient.enterLocalPassiveMode();   
        }   
        setFileType(ftpClient); 

        try {   
            ftpClient.setSoTimeout(clientTimeout);   
        } catch (SocketException e) {   
            throw new IOException("Set timeout error.", e);   
        }   

        return ftpClient;   
    }   

    private void setFileType(FTPClient ftpClient) throws IOException {   
        try {   
            if (binaryTransfer) {   
                ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);   
            } else {   
                ftpClient.setFileType(FTPClient.ASCII_FILE_TYPE);   
            }   
        } catch (IOException e) {   
            throw new IOException("Could not to set file type.", e);   
        }   
    }   

    public boolean connect(FTPClient ftpClient) throws IOException {   
        try {   
            ftpClient.connect(host, port);   

            int reply = ftpClient.getReplyCode();   

            if (FTPReply.isPositiveCompletion(reply)) {   
                if (ftpClient.login(username, password)) {   
                    setFileType(ftpClient);   
                    return true;   
                }   
            } else {   
                this.ftpClient.disconnect();   
                throw new IOException("FTP server refused connection.");   
            }   
        } catch (IOException e) {   
            if (this.ftpClient.isConnected()) {   
                try {   
                    this.ftpClient.disconnect();
                } catch (IOException e1) {   
                    throw new IOException("Could not disconnect from server.", e);   
                }   

            }   
            throw new IOException("Could not connect to server.", e);   
        }   
        return false;   
    }   

    private void disconnect() throws IOException {   
        try {   
            this.ftpClient.logout();   
        } catch (IOException e) {   
            System.out.println("logout may timeout!");
        } finally {
            if (this.ftpClient.isConnected()) {   
                this.ftpClient.disconnect();   
            }   
        }  
    }   

    public InputStream getStream(String serverFile) throws IOException {
        InputStream inStream = null;
        try {
            inStream = this.ftpClient.retrieveFileStream(serverFile);
            System.out.println("inStream get over!");
            return inStream;
        } catch (IOException e) {
            System.out.println("get stream exception");
            return null;
        }
    }

    public boolean writeStream(InputStream input, String localFile) throws IOException {
        FileOutputStream fout = new FileOutputStream(localFile);
        int ch = 0;
        if(input == null){
            System.out.println("input is null");
            return false;
        }
        try {
            ch = input.read();
            while(ch != -1){
                fout.write(ch);
                ch = input.read();
            }
            System.out.println("write over!");
            return flag;
        } catch (IOException e) {
            throw new IOException("Couldn't get file from server.", e);
        } 
    }

    public boolean isExist(String remoteFilePath)throws IOException{

        try{
            File file=new File(remoteFilePath);

            String remotePath=remoteFilePath.substring(0,(remoteFilePath.indexOf(file.getName())-1));
            String[] listNames = this.ftpClient.listNames(remotePath);   
            System.out.println(remoteFilePath);
            for(int i=0;i<listNames.length;i++){
                System.out.println(listNames[i]);
                if(remoteFilePath.equals(listNames[i])){
                    flag=true;
                    System.out.println("file:"+file.getName()+" existed");
                    break;

                }else {
                    flag=false;
                }
            }

        } catch (IOException e) {   
            throw new IOException("FILE EXCEPTION", e);   
        } 
        return flag;
    }

    //main for testing
    public static void main(String[] args){
        HDFSUtil hdfsutil = new HDFSUtil("hdfs://nj01-nanling-hdfs.dmop.baidu.com:54310");
        String hdfspath = "/app/ps/spider/wdmqa/wangweilong/test/";
        String filename = "HDFSUtil.java";
        String localname = "/home/work/workspace/project/dhc2-0/dhc/base/ftp/HDFSUtil.java";
        hdfsutil.setHdfsPath(hdfspath);
        hdfsutil.setFilePath(hdfsutil.getHdfsNode()+hdfsutil.getHdfsPath()+filename);
        System.out.println(hdfsutil.getHdfsNode());
        System.out.println(hdfsutil.getHdfsPath());
        System.out.println(hdfsutil.getFilePath());
        try {
            hdfsutil.upLoad(localname, hdfsutil.getFilePath());
            //hdfsutil.downLoad(localname, hdfsutil.getFilePath());
        } catch (IOException e){
            System.out.println("AAAAA");
        }
    }

}

运行方式是:

javac -classpath $CLASSPATH:$HADOOPCLASS HDFSUtil

最后在 hdfsutil.upLoad 异常,打印 AAAAA

本人 Java 不是太懂,临时需要来做的需求,代码上应该没错,照着书上写的,但是就是不知道为什么运行会异常,请大家帮忙解答一下,谢谢!


1个回答

︿ 0

不看错误提示如何找错误。。。

System.out.println("AAAAA"); 改为 e.printStackTrace();看下抛出的异常具体是啥